WIP: Web scraping & automated data update

backend/package.json

@@ -7,6 +7,9 @@
     "add_buildings": "ts-node scripts/add_buildings.ts",
     "add_landlords": "ts-node scripts/add_landlords.ts",
     "add_reviews": "ts-node scripts/add_reviews_nodups.ts",
+    "export_apartments": "env-cmd -f ../.env.prod ts-node scripts/export_apartments.ts",
+    "update_apartments": "env-cmd -f ../.env.prod ts-node scripts/update_apartments_from_csv.ts",
+    "export_apartment_names": "ts-node scripts/export_apartment_names.ts",
     "build": "tsc",
     "tsc": "tsc",
     "start": "node dist/backend/src/server.js",
@@ -21,6 +24,7 @@
   "dependencies": {
     "@firebase/testing": "^0.20.11",
     "axios": "^0.21.1",
+    "cheerio": "1.0.0-rc.10",
     "cors": "^2.8.5",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
@@ -33,10 +37,11 @@
     "typescript": "^4.0.5"
   },
   "devDependencies": {
+    "@types/cheerio": "0.22.31",
     "@types/cors": "^2.8.10",
     "@types/express": "^4.17.11",
     "@types/morgan": "^1.9.2",
     "@types/supertest": "^2.0.11",
     "eslint-import-resolver-typescript": "^2.5.0"
   }
-}
+}

backend/scripts/add_buildings.ts

@@ -43,13 +43,11 @@ const formatBuilding = ({
   name,
   address,
   landlordId: landlordId.toString(),
-  numBaths: 0,
-  numBeds: 0,
+  roomTypes: [], // Initialize with empty room types
   photos: [],
   area: getAreaType(area),
   latitude,
   longitude,
-  price: 0,
   distanceToCampus: 0,
 });
 

backend/scripts/export_apartment_names.ts

@@ -0,0 +1,89 @@
+import axios from 'axios';
+import * as fs from 'fs';
+import * as path from 'path';
+
+// Define the structure of the API response based on the AdminPage usage
+interface ApartmentData {
+  buildingData: {
+    id: string;
+    name: string;
+    address: string;
+    area: string;
+    numBeds: number;
+    numBaths: number;
+    landlordId: string;
+    photos: string[];
+    urlName: string;
+    latitude: number;
+    longitude: number;
+    distanceToCampus: number;
+  };
+  numReviews: number;
+  company: string;
+  avgRating: number;
+  avgPrice: number;
+}
+
+interface ApiResponse {
+  buildingData: ApartmentData[];
+  isEnded: boolean;
+}
+
+/**
+ * Script to export apartment names to a CSV file
+ *
+ * This script fetches all apartment data from the API endpoint used in AdminPage
+ * and exports just the apartment names to a CSV file.
+ */
+const exportApartmentNames = async () => {
+  try {
+    console.log('Fetching apartment data...');
+
+    // Use the same API endpoint as AdminPage
+    const response = await axios.get<ApiResponse>(
+      'http://localhost:8080/api/page-data/home/1000/numReviews'
+    );
+
+    if (!response.data || !response.data.buildingData) {
+      throw new Error('No apartment data received from API');
+    }
+
+    const apartments = response.data.buildingData;
+    console.log(`Found ${apartments.length} apartments`);
+
+    // Extract apartment names
+    const apartmentNames = apartments.map((apt) => apt.buildingData.name);
+
+    // Create CSV content
+    const csvHeader = 'Apartment Name\n';
+    const csvContent = apartmentNames.map((name) => `"${name}"`).join('\n');
+    const fullCsvContent = csvHeader + csvContent;
+
+    // Write to CSV file
+    const outputPath = path.join(__dirname, 'apartment_names.csv');
+    fs.writeFileSync(outputPath, fullCsvContent, 'utf8');
+
+    console.log(`Successfully exported ${apartmentNames.length} apartment names to: ${outputPath}`);
+    console.log('First few apartment names:');
+    apartmentNames.slice(0, 5).forEach((name, index) => {
+      console.log(`${index + 1}. ${name}`);
+    });
+  } catch (error) {
+    console.error('Error exporting apartment names:', error);
+
+    if (axios.isAxiosError(error)) {
+      if (error.code === 'ECONNREFUSED') {
+        console.error(
+          'Could not connect to the server. Make sure the backend server is running on localhost:3000'
+        );
+      } else {
+        console.error('API request failed:', error.message);
+      }
+    }
+
+    process.exit(1);
+  }
+};
+
+// Run the script
+exportApartmentNames();

backend/scripts/export_apartments.ts

@@ -0,0 +1,101 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { db } from '../src/firebase-config';
+
+/**
+ * export_apartments.ts
+ *
+ * Exports all apartment documents from Firestore to a CSV file.
+ * The business team can then edit the CSV and pass it to update_apartments_from_csv.ts.
+ *
+ * Usage:
+ *   env-cmd -f ../.env.prod ts-node scripts/export_apartments.ts
+ *
+ * Output:
+ *   backend/scripts/apartments_export.csv
+ */
+
+const buildingCollection = db.collection('buildings');
+
+// TODO: might have to change header fields or order (talk with business)
+const CSV_HEADERS = [
+  'id',
+  'name',
+  'address',
+  'landlordId',
+  'numBeds',
+  'numBaths',
+  'price',
+  'area',
+  'latitude',
+  'longitude',
+  'distanceToCampus',
+];
+
+// escape CSV field values
+export const escapeCSVField = (value: unknown): string => {
+  const str = value === null || value === undefined ? '' : String(value);
+  if (str.includes(',') || str.includes('"') || str.includes('\n')) {
+    return `"${str.replace(/"/g, '""')}"`;
+  }
+  return str;
+};
+
+const exportApartments = async () => {
+  console.log('Fetching apartments from Firestore...');
+
+  const snapshot = await buildingCollection.get();
+
+  if (snapshot.empty) {
+    console.log('No apartments found in the database.');
+    process.exit(0);
+  }
+
+  console.log(`Found ${snapshot.docs.length} apartments. Writing CSV...`);
+
+  const rows: string[] = [CSV_HEADERS.join(',')];
+
+  snapshot.docs.forEach((doc) => {
+    const data = doc.data();
+    const row = [
+      doc.id,
+      data.name,
+      data.address,
+      data.landlordId,
+      data.numBeds,
+      data.numBaths,
+      data.price,
+      data.area,
+      data.latitude,
+      data.longitude,
+      data.distanceToCampus,
+    ]
+      .map(escapeCSVField)
+      .join(',');
+
+    rows.push(row);
+  });
+
+  const outputPath = path.join(__dirname, 'apartments_export.csv');
+  fs.writeFileSync(outputPath, rows.join('\n'), 'utf8');
+
+  console.log(`Export complete: ${outputPath}`);
+  console.log(`   ${snapshot.docs.length} apartments exported.`);
+  console.log('');
+  console.log('Next steps:');
+  console.log('  1. Open apartments_export.csv in Excel or Google Sheets');
+  console.log('  2. Edit the fields you want to update (do NOT change the id column)');
+  console.log('  3. Save as CSV');
+  console.log('  4. Run: env-cmd -f ../.env.prod ts-node scripts/update_apartments_from_csv.ts');
+
+  process.exit(0);
+};
+
+if (require.main === module) {
+  exportApartments().catch((err) => {
+    console.error('Export failed:', err);
+    process.exit(1);
+  });
+}
+
+export default exportApartments;