initial install

teressa-copeland-homes/scripts/scrape-skyslope-forms.ts

@@ -13,6 +13,8 @@
 import { chromium } from 'playwright';
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
+import { tmpdir } from 'node:os';
+import AdmZip from 'adm-zip';
 import { config } from 'dotenv';
 
 config({ path: path.resolve(process.cwd(), '.env.local') });
@@ -221,6 +223,17 @@ async function handleNRDSAuth(page: import('playwright').Page) {
   }
 }
 
+function extractFormNames(bodyText: string): string[] {
+  const lines = bodyText.split('\n').map(l => l.trim()).filter(l => l.length > 0);
+  const formNames: string[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (lines[i] === 'Add' && i > 0 && lines[i - 1] !== 'Add' && lines[i - 1].length > 3) {
+      formNames.push(lines[i - 1]);
+    }
+  }
+  return formNames;
+}
+
 async function downloadFormsInView(
   page: import('playwright').Page,
   context: import('playwright').BrowserContext,
@@ -228,19 +241,35 @@ async function downloadFormsInView(
   downloaded: string[],
   failed: string[]
 ) {
-  // Flow: click form name → preview opens → click Download button → save file
+  // Flow: scroll to load all forms, then click form name → preview → Download button → save
 
-  // Extract form names from the page body text — the list renders as "Name\nAdd\nName\nAdd..."
-  const bodyText = await page.locator('body').innerText().catch(() => '');
-  const lines = bodyText.split('\n').map(l => l.trim()).filter(l => l.length > 3);
-  const formNames: string[] = [];
-  for (let i = 0; i < lines.length; i++) {
-    if (lines[i] === 'Add' && i > 0 && lines[i - 1] !== 'Add' && lines[i - 1].length > 3) {
-      formNames.push(lines[i - 1]);
+  // Scroll down repeatedly to trigger infinite scroll, collecting all form names
+  const allNames = new Set<string>();
+  let prevCount = 0;
+  let stallRounds = 0;
+  while (stallRounds < 5) {
+    // Scroll both window and any inner scrollable container to handle virtualized lists
+    await page.evaluate(() => {
+      window.scrollTo(0, document.body.scrollHeight);
+      const scrollable = document.querySelector('[class*="scroll"], [class*="list"], main, [role="main"], .overflow-auto, .overflow-y-auto');
+      if (scrollable) scrollable.scrollTop = scrollable.scrollHeight;
+    });
+    await page.waitForTimeout(2000);
+    const bodyText = await page.locator('body').innerText().catch(() => '');
+    for (const n of extractFormNames(bodyText)) allNames.add(n);
+    if (allNames.size === prevCount) {
+      stallRounds++;
+    } else {
+      stallRounds = 0;
+      prevCount = allNames.size;
+      process.stdout.write(`  Loaded ${allNames.size} forms so far...\n`);
     }
   }
-  const names = [...new Set(formNames)];
+  // Scroll back to top before clicking
+  await page.evaluate(() => window.scrollTo(0, 0));
+  await page.waitForTimeout(500);
 
+  const names = [...allNames];
   console.log(`  Found ${names.length} forms to download`);
   if (names.length === 0) {
     await page.screenshot({ path: `scripts/debug-no-forms-${Date.now()}.png` });
@@ -285,7 +314,9 @@ async function downloadFormsInView(
         page.waitForEvent('download', { timeout: 20_000 }),
         downloadBtn.click(),
       ]);
-      await download.saveAs(destPath);
+      const tmpPath = path.join(tmpdir(), `skyslope-${Date.now()}.tmp`);
+      await download.saveAs(tmpPath);
+      await savePdf(tmpPath, destPath);
       process.stdout.write(`  ✓ ${sanitized}.pdf\n`);
       downloaded.push(sanitized);
     } catch (err) {
@@ -379,6 +410,21 @@ async function interceptPdfOnClick(
   });
 }
 
+/** If the downloaded file is a ZIP, extract the first PDF inside; otherwise move as-is. */
+async function savePdf(tmpPath: string, destPath: string) {
+  const buf = await fs.readFile(tmpPath);
+  const isPk = buf[0] === 0x50 && buf[1] === 0x4b; // PK magic bytes = ZIP
+  if (isPk) {
+    const zip = new AdmZip(buf);
+    const entry = zip.getEntries().find(e => e.entryName.toLowerCase().endsWith('.pdf'));
+    if (!entry) throw new Error('ZIP contained no PDF entry');
+    await fs.writeFile(destPath, entry.getData());
+  } else {
+    await fs.rename(tmpPath, destPath);
+  }
+  await fs.unlink(tmpPath).catch(() => {}); // clean up tmp if rename didn't move it
+}
+
 main().catch(err => {
   console.error('Fatal:', err.message);
   process.exit(1);