import { NextResponse } from "next/server";

// POST - Scrape product from URL
export async function POST(request: Request) {
  try {
    const body = await request.json();
    const { url } = body;

    if (!url || typeof url !== "string") {
      return NextResponse.json({ error: "URL is required" }, { status: 400 });
    }

    console.log(`[Scraper] Attempting to scrape: ${url}`);

    // Fetch the HTML content
    let html: string = "";
    try {
      // Try multiple methods to fetch the page
      const proxies = [
        `https://api.allorigins.win/get?url=${encodeURIComponent(url)}`,
        `https://corsproxy.io/?${encodeURIComponent(url)}`,
        `https://api.codetabs.com/v1/proxy?quest=${encodeURIComponent(url)}`,
      ];

      let lastError: Error | null = null;
      let fetchSuccess = false;

      // Try direct fetch first with enhanced headers
      try {
        console.log("[Scraper] Trying direct fetch...");
        const response = await fetch(url, {
          headers: {
            "User-Agent":
              "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.9",
            "Accept-Encoding": "gzip, deflate, br",
            "Connection": "keep-alive",
            "Upgrade-Insecure-Requests": "1",
            "Sec-Fetch-Dest": "document",
            "Sec-Fetch-Mode": "navigate",
            "Sec-Fetch-Site": "none",
            "Cache-Control": "max-age=0",
          },
          // Add timeout
          signal: AbortSignal.timeout(30000), // 30 second timeout
        });

        if (response.ok) {
          html = await response.text();
          console.log(`[Scraper] Direct fetch successful, HTML length: ${html.length}`);
          fetchSuccess = true;
        } else {
          throw new Error(`HTTP ${response.status}: ${response.statusText}`);
        }
      } catch (directError) {
        console.log(`[Scraper] Direct fetch failed: ${directError instanceof Error ? directError.message : "Unknown error"}`);
        lastError = directError instanceof Error ? directError : new Error("Direct fetch failed");

        // Try proxies
        for (const proxyUrl of proxies) {
          try {
            console.log(`[Scraper] Trying proxy: ${proxyUrl.substring(0, 50)}...`);
            const proxyResponse = await fetch(proxyUrl, {
              headers: {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
              },
              signal: AbortSignal.timeout(30000),
            });

            if (proxyResponse.ok) {
              const proxyData = await proxyResponse.json();
              html = proxyData.contents || proxyData.data || "";
              
              // If proxy returns text directly
              if (!html && typeof proxyData === "string") {
                html = proxyData;
              }

              if (html && html.length > 100) {
                console.log(`[Scraper] Proxy fetch successful, HTML length: ${html.length}`);
                fetchSuccess = true;
                break;
              }
            }
          } catch (proxyError) {
            console.log(`[Scraper] Proxy failed: ${proxyError instanceof Error ? proxyError.message : "Unknown error"}`);
            continue;
          }
        }
      }

      if (!fetchSuccess || !html || html.length < 100) {
        throw new Error(
          `Failed to fetch meaningful content. HTML length: ${html?.length || 0}. ` +
          `Last error: ${lastError?.message || "Unknown"}`
        );
      }

      console.log(`[Scraper] Successfully fetched HTML, length: ${html.length}`);
    } catch (error) {
      console.error("[Scraper] Error fetching URL:", error);
      return NextResponse.json(
        {
          error: "Failed to fetch the URL",
          details: error instanceof Error ? error.message : "Unknown error",
          suggestion: "The website may be blocking automated requests or require JavaScript to load content. Try a different URL or contact support.",
        },
        { status: 400 }
      );
    }

    // Extract data using improved patterns
    console.log("[Scraper] Extracting product data...");
    
    // Always try both standard and alternative extraction methods for maximum image coverage
    const standardImages = extractImages(html, url);
    console.log(`[Scraper] Found ${standardImages.length} images using standard extraction`);
    
    // Always try alternative extraction to get more images
    console.log("[Scraper] Trying alternative image extraction methods...");
    const altImages = extractImagesAlternative(html, url);
    console.log(`[Scraper] Found ${altImages.length} images using alternative extraction`);
    
    // Combine and deduplicate images (prioritize standard extraction results)
    const allImages = [...new Set([...standardImages, ...altImages])];
    console.log(`[Scraper] Found ${allImages.length} total unique images`);
    
    let scrapedData: any = {
      title: extractTitle(html, url),
      price: extractPrice(html, url),
      images: allImages,
      shortDescription: extractShortDescription(html),
      fullDescription: extractFullDescription(html),
      productDetails: extractProductDetails(html, url),
    };

    // Brand append in title
    if (scrapedData.title) {
      const brandSuffix = "Server Zone Computer";
      if (!scrapedData.title.toLowerCase().includes(brandSuffix.toLowerCase())) {
        scrapedData.title = `${scrapedData.title} | ${brandSuffix}`;
      }
    }

    // Image alt generation
    const altBaseTitle = scrapedData.title || extractTitleFromUrl(url) || "Product";
    const altString = `${altBaseTitle} - Wholesale Dubai - Server Zone Computer`;
    scrapedData.imageObjects = (allImages || []).map((src: string) => ({ src, alt: altString }));

    // Infer variations: color, meters, storage HDD, RAM
    const sourceText = [
      scrapedData.title || "",
      scrapedData.shortDescription || "",
      scrapedData.fullDescription || "",
      JSON.stringify(scrapedData.productDetails || {})
    ].join(" \n ").toLowerCase();

    const colorList = Array.from(new Set((sourceText.match(/\b(black|white|silver|gray|grey|gold|blue|red|green|yellow|purple|pink|beige|brown|orange)\b/g) || []).map(s=>s.charAt(0).toUpperCase()+s.slice(1))));
    const metersList = Array.from(new Set((sourceText.match(/\b(\d+(?:\.\d+)?)\s*(m|meter|meters|mtr)\b/g) || []).map(s=>s.replace(/\b(meter|meters|mtr)\b/gi,'m'))));
    const hddList = Array.from(new Set((sourceText.match(/\b(\d+)\s*(tb|gb)\s*(hdd|hard ?drive)\b/g) || []).map(s=>s.toUpperCase())));
    const ramList = Array.from(new Set((sourceText.match(/\b(\d+)\s*gb\s*ram\b/g) || []).map(s=>s.toUpperCase())));

    scrapedData.variations = {
      color: colorList,
      meters: metersList,
      storageHDD: hddList,
      ram: ramList,
    };

    // If title is empty, try to extract from URL (for Microless and similar)
    if (!scrapedData.title) {
      const urlTitle = extractTitleFromUrl(url);
      if (urlTitle) {
        scrapedData.title = urlTitle;
        console.log("[Scraper] Title extracted from URL:", urlTitle);
      }
    }

    // Enhance product details with URL-based extraction for Microless
    if (url.includes("microless.com")) {
      const urlDetails = extractProductDetailsFromUrl(url);
      if (urlDetails && Object.keys(urlDetails).length > 0) {
        scrapedData.productDetails = {
          ...(scrapedData.productDetails || {}),
          ...urlDetails,
        };
        console.log("[Scraper] Enhanced product details from URL");
      }
    }

    console.log("[Scraper] Extraction complete:", {
      title: scrapedData.title ? "Found" : "Not found",
      price: scrapedData.price ? "Found" : "Not found",
      images: scrapedData.images?.length || 0,
      description: scrapedData.shortDescription ? "Found" : "Not found",
      details: scrapedData.productDetails ? Object.keys(scrapedData.productDetails).length + " keys" : "Not found",
    });

    // If we couldn't extract much, still return what we have but with a warning
    if (!scrapedData.title && !scrapedData.price && scrapedData.images.length === 0) {
      return NextResponse.json({
        success: false,
        error: "Limited product information extracted",
        message: "The website may require JavaScript to load content. Some information was extracted from the URL.",
        data: scrapedData,
      });
    }

    return NextResponse.json({
      success: true,
      data: scrapedData,
    });
  } catch (error) {
    console.error("[Scraper] Error scraping product:", error);
    return NextResponse.json(
      {
        error: "Failed to scrape product",
        details: error instanceof Error ? error.message : "Unknown error",
      },
      { status: 500 }
    );
  }
}

// Helper functions for extraction
function extractTitle(html: string, url: string): string {
  // Try JSON-LD first (most reliable for e-commerce)
  try {
    const jsonLdPattern = /<script[^>]*type=["']application\/ld\+json["'][^>]*>(.*?)<\/script>/gis;
    const matches = html.matchAll(jsonLdPattern);
    for (const match of matches) {
      try {
        const jsonData = JSON.parse(match[1]);
        if (jsonData["@type"] === "Product" && jsonData.name) {
          return cleanText(jsonData.name);
        }
        if (jsonData.name) {
          return cleanText(jsonData.name);
        }
      } catch (e) {
        // Continue to next pattern
      }
    }
  } catch (e) {
    // Continue to other patterns
  }

  // Try multiple patterns for Microless and other sites
  const patterns = [
    // Microless specific patterns
    /<h1[^>]*class=["'][^"']*product-title[^"']*["'][^>]*>(.*?)<\/h1>/is,
    /<h1[^>]*class=["'][^"']*product[^"']*name[^"']*["'][^>]*>(.*?)<\/h1>/is,
    /product-title[^>]*>(.*?)<\/h1>/is,
    // Generic patterns
    /<h1[^>]*>(.*?)<\/h1>/is,
    /og:title["\s]*content=["'](.*?)["']/i,
    /twitter:title["\s]*content=["'](.*?)["']/i,
    /<title>(.*?)<\/title>/i,
    /class=["']product.*?title["'][^>]*>(.*?)</is,
    /data-product-name=["'](.*?)["']/i,
    /itemprop=["']name["'][^>]*content=["'](.*?)["']/i,
  ];

  for (const pattern of patterns) {
    const match = html.match(pattern);
    if (match && match[1]) {
      const title = cleanText(match[1]);
      if (title && title.length > 3 && title.length < 200) {
        return title;
      }
    }
  }

  // Try to extract from URL as last resort
  try {
    const urlParts = url.split("/");
    const lastPart = urlParts[urlParts.length - 1];
    if (lastPart && lastPart.length > 10) {
      return lastPart
        .split("-")
        .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
        .join(" ");
    }
  } catch (e) {
    // Ignore
  }

  return "";
}

function extractPrice(html: string, url: string): string {
  // Try JSON-LD first (most reliable)
  try {
    const jsonLdPattern = /<script[^>]*type=["']application\/ld\+json["'][^>]*>(.*?)<\/script>/gis;
    const matches = html.matchAll(jsonLdPattern);
    for (const match of matches) {
      try {
        const jsonData = JSON.parse(match[1]);
        if (jsonData["@type"] === "Product" && jsonData.offers) {
          const offer = Array.isArray(jsonData.offers) ? jsonData.offers[0] : jsonData.offers;
          if (offer.price) {
            const price = typeof offer.price === "string" ? parseFloat(offer.price) : offer.price;
            const currency = offer.priceCurrency || "USD";
            if (price && price > 0) {
              return formatPrice(price, currency);
            }
          }
        }
        if (jsonData.price) {
          const price = typeof jsonData.price === "string" ? parseFloat(jsonData.price) : jsonData.price;
          const currency = jsonData.priceCurrency || "USD";
          if (price && price > 0) {
            return formatPrice(price, currency);
          }
        }
      } catch (e) {
        // Continue to next pattern
      }
    }
  } catch (e) {
    // Continue to other patterns
  }

  // Try multiple patterns for different e-commerce sites
  const pricePatterns = [
    // Microless specific patterns
    /price["\s]*:["\s]*["']?([\d,]+\.?\d*)/i,
    /class=["'][^"']*price[^"']*["'][^>]*>.*?([\d,]+\.?\d*)/is,
    /data-price=["']([\d,]+\.?\d*)["']/i,
    /itemprop=["']price["'][^>]*content=["']([\d,]+\.?\d*)["']/i,
    // Currency patterns (AED, USD, EUR, etc.)
    /AED\s*([\d,]+\.?\d*)/i,
    /USD\s*([\d,]+\.?\d*)/i,
    /EUR\s*([\d,]+\.?\d*)/i,
    /[\$£€¥AED]?\s*([\d,]+\.?\d*)/g,
    // Generic price patterns
    /"price":\s*["']?([\d,]+\.?\d*)/i,
    /"price":\s*(\d+\.?\d*)/i,
    /price["\s]*content=["']([\d,]+\.?\d*)["']/i,
  ];

  const prices: Array<{ price: number; currency: string }> = [];
  const currencySymbols: { [key: string]: string } = {
    AED: "AED",
    USD: "$",
    EUR: "€",
    GBP: "£",
    SAR: "SAR",
  };

  for (const pattern of pricePatterns) {
    const matches = html.matchAll(pattern);
    for (const match of matches) {
      const priceStr = match[1]?.replace(/,/g, "");
      if (!priceStr) continue;

      const price = parseFloat(priceStr);
      if (price && price > 0 && price < 1000000) {
        // Try to detect currency
        let currency = "USD";
        const matchText = match[0] || "";
        if (matchText.includes("AED")) currency = "AED";
        else if (matchText.includes("USD") || matchText.includes("$")) currency = "USD";
        else if (matchText.includes("EUR") || matchText.includes("€")) currency = "EUR";
        else if (matchText.includes("GBP") || matchText.includes("£")) currency = "GBP";
        else if (matchText.includes("SAR")) currency = "SAR";

        prices.push({ price, currency });
      }
    }
  }

  if (prices.length > 0) {
    // Sort by price and get median
    prices.sort((a, b) => a.price - b.price);
    const median = prices[Math.floor(prices.length / 2)];
    return formatPrice(median.price, median.currency);
  }

  return "";
}

function formatPrice(price: number, currency: string = "USD"): string {
  const symbols: { [key: string]: string } = {
    AED: "AED ",
    USD: "$",
    EUR: "€",
    GBP: "£",
    SAR: "SAR ",
  };
  const symbol = symbols[currency] || currency + " ";
  return `${symbol}${price.toFixed(2)}`;
}

function extractImages(html: string, baseUrl: string): string[] {
  const images: string[] = [];
  const seen = new Set<string>();
  const imageCandidates: Array<{ url: string; priority: number }> = [];

  // Try JSON-LD for images (most reliable) - Highest priority
  try {
    const jsonLdPattern = /<script[^>]*type=["']application\/ld\+json["'][^>]*>(.*?)<\/script>/gis;
    const matches = html.matchAll(jsonLdPattern);
    for (const match of matches) {
      try {
        const jsonData = JSON.parse(match[1]);
        if (jsonData["@type"] === "Product") {
          if (jsonData.image) {
            const imageUrls = Array.isArray(jsonData.image) ? jsonData.image : [jsonData.image];
            imageUrls.forEach((img: string) => {
              const absoluteUrl = makeAbsoluteUrl(img, baseUrl);
              if (absoluteUrl && !seen.has(absoluteUrl)) {
                imageCandidates.push({ url: absoluteUrl, priority: 10 }); // Highest priority
                seen.add(absoluteUrl);
              }
            });
          }
        }
      } catch (e) {
        // Continue to next pattern
      }
    }
  } catch (e) {
    // Continue to other patterns
  }

  // Try to find product gallery containers first (high priority)
  const galleryPatterns = [
    /<div[^>]*class=["'][^"']*product-gallery[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi,
    /<div[^>]*class=["'][^"']*product-images[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi,
    /<div[^>]*class=["'][^"']*gallery[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi,
    /<div[^>]*class=["'][^"']*product-slider[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi,
    /<ul[^>]*class=["'][^"']*product-thumbnails[^"']*["'][^>]*>([\s\S]*?)<\/ul>/gi,
    /<div[^>]*class=["'][^"']*swiper-wrapper[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi, // Swiper slider
    /<div[^>]*class=["'][^"']*image-gallery[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi,
    /<div[^>]*id=["']product-images["'][^>]*>([\s\S]*?)<\/div>/gi,
    /<div[^>]*id=["']product-gallery["'][^>]*>([\s\S]*?)<\/div>/gi,
  ];

  for (const galleryPattern of galleryPatterns) {
    const galleryMatches = html.matchAll(galleryPattern);
    for (const galleryMatch of galleryMatches) {
      const galleryHtml = galleryMatch[1];
      const imgPatterns = [
        /<img[^>]+(?:src|data-src|data-lazy-src|data-original)=["']([^"']+)["']/gi,
        /<img[^>]+(?:src|data-src|data-lazy-src|data-original)=["']([^"']+)["']/gi,
      ];
      
      for (const imgPattern of imgPatterns) {
        const imgMatches = galleryHtml.matchAll(imgPattern);
        for (const imgMatch of imgMatches) {
          let imgUrl = imgMatch[1];
          if (!imgUrl) continue;
          
          const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
          if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
            imageCandidates.push({ url: absoluteUrl, priority: 8 });
            seen.add(absoluteUrl);
          }
        }
      }
    }
  }

  // Try multiple patterns for images (medium priority)
  const patterns = [
    // Lazy loaded images (common in e-commerce) - High priority
    { pattern: /<img[^>]+data-src=["']([^"']+)["']/gi, priority: 7 },
    { pattern: /<img[^>]+data-lazy-src=["']([^"']+)["']/gi, priority: 7 },
    { pattern: /<img[^>]+data-original=["']([^"']+)["']/gi, priority: 7 },
    { pattern: /<img[^>]+data-zoom-image=["']([^"']+)["']/gi, priority: 7 },
    { pattern: /<img[^>]+data-large-image=["']([^"']+)["']/gi, priority: 7 },
    { pattern: /<img[^>]+data-full-image=["']([^"']+)["']/gi, priority: 7 },
    // Microless and similar sites - specific patterns
    { pattern: /<img[^>]+data-image-src=["']([^"']+)["']/gi, priority: 7 },
    { pattern: /<img[^>]+data-img=["']([^"']+)["']/gi, priority: 7 },
    { pattern: /<img[^>]+data-product-image=["']([^"']+)["']/gi, priority: 7 },
    // Open Graph and Twitter images
    { pattern: /og:image["\s]*content=["']([^"']+)["']/gi, priority: 6 },
    { pattern: /twitter:image["\s]*content=["']([^"']+)["']/gi, priority: 6 },
    // Product image classes
    { pattern: /class=["'][^"']*product-image[^"']*["'][^>]*(?:src|data-src)=["']([^"']+)["']/gi, priority: 6 },
    { pattern: /class=["'][^"']*product-img[^"']*["'][^>]*(?:src|data-src)=["']([^"']+)["']/gi, priority: 6 },
    { pattern: /class=["'][^"']*main-image[^"']*["'][^>]*(?:src|data-src)=["']([^"']+)["']/gi, priority: 6 },
    { pattern: /class=["'][^"']*featured-image[^"']*["'][^>]*(?:src|data-src)=["']([^"']+)["']/gi, priority: 6 },
    // Data attributes
    { pattern: /data-image=["']([^"']+)["']/gi, priority: 5 },
    { pattern: /data-product-image=["']([^"']+)["']/gi, priority: 6 },
    // Background images in style attributes
    { pattern: /style=["'][^"']*background-image:\s*url\(["']?([^"')]+)["']?\)/gi, priority: 4 },
    // Source tags in picture elements
    { pattern: /<source[^>]+srcset=["']([^"']+)["']/gi, priority: 5 },
    // Generic img tags (lower priority, but check for product context)
    { pattern: /<img[^>]+src=["']([^"']+)["']/gi, priority: 3 },
    // Srcset
    { pattern: /srcset=["']([^"']+)["']/gi, priority: 2 },
  ];

  for (const { pattern, priority } of patterns) {
    const matches = html.matchAll(pattern);
    for (const match of matches) {
      let imgUrl = match[1];
      if (!imgUrl) continue;

      // Handle srcset (take first and largest URL)
      if (imgUrl.includes(",")) {
        const srcsetUrls = imgUrl.split(",").map(s => s.trim().split(" ")[0]);
        // Prefer larger images (usually last in srcset)
        imgUrl = srcsetUrls[srcsetUrls.length - 1] || srcsetUrls[0];
      }

      // Handle background-image URLs
      if (imgUrl.includes("url(")) {
        imgUrl = imgUrl.replace(/.*url\(["']?/, "").replace(/["']?\).*/, "");
      }

      const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
      if (!absoluteUrl || seen.has(absoluteUrl)) continue;

      if (isProductImage(absoluteUrl)) {
        imageCandidates.push({ url: absoluteUrl, priority });
        seen.add(absoluteUrl);
      }
    }
  }

  // Sort by priority (highest first), then by URL patterns that indicate larger images
  imageCandidates.sort((a, b) => {
    if (b.priority !== a.priority) return b.priority - a.priority;
    
    // Within same priority, prefer larger/higher quality images
    const aLarge = a.url.includes("large") || a.url.includes("original") || a.url.includes("zoom") || a.url.includes("hd");
    const bLarge = b.url.includes("large") || b.url.includes("original") || b.url.includes("zoom") || b.url.includes("hd");
    
    if (aLarge && !bLarge) return -1;
    if (!aLarge && bLarge) return 1;
    return 0;
  });

  // Extract URLs and remove duplicates
  const uniqueImages = Array.from(new Set(imageCandidates.map(c => c.url)));
  
  // Return up to 15 images (increased from 10)
  return uniqueImages.slice(0, 15);
}

function isProductImage(url: string): boolean {
  if (!url) return false;
  
  const urlLower = url.toLowerCase();
  
  // Exclude non-product images - more comprehensive list
  const excludePatterns = [
    "icon", "logo", "avatar", "sprite", "placeholder", "banner", "ad", "advertisement",
    "button", "badge", "flag", "social", "facebook", "twitter", "instagram", "youtube",
    "loading", "spinner", "arrow", "chevron", "close", "menu", "hamburger",
    "1x1", "pixel", "tracking", "analytics", "beacon", "favicon", "thumbnail-",
    "thumb-", "small-", "tiny-", "mini-", "preview-", "watermark", "stamp",
  ];
  
  // Check if URL contains exclude patterns (but allow if it's clearly a product image)
  let hasExcludePattern = false;
  for (const pattern of excludePatterns) {
    if (urlLower.includes(pattern)) {
      // Allow if it's part of a larger product image path
      if (!urlLower.match(new RegExp(`(${pattern}[^/]*product|product[^/]*${pattern})`, 'i'))) {
        hasExcludePattern = true;
        break;
      }
    }
  }
  
  // Must be an image file
  const hasImageExtension = url.match(/\.(jpg|jpeg|png|webp|gif|svg|bmp|tiff|jfif)(\?|$)/i);
  if (!hasImageExtension) return false;
  
  // Reject obvious small images based on tokens/params (e.g., 80x80, w=120, _SX38_)
  const smallDimensionTokens = [
    /(^|[^\d])([1-2]?\d{1,2})x([1-2]?\d{1,2})([^\d]|$)/,     // 50x50, 150x150, 200x120
    /[?&](w|width|h|height)=([1-2]?\d{1,2})/i,                 // w=120, height=100
    /_(sx|sy|ux|uy)(\d{1,3})_/i,                               // _SX38_
  ];
  for (const rx of smallDimensionTokens) {
    const m = urlLower.match(rx);
    if (m) return false; // likely a thumbnail
  }
  
  // If it has exclude pattern but no product context, exclude it
  if (hasExcludePattern && !urlLower.includes("product") && !urlLower.includes("item")) {
    return false;
  }
  
  // Product image indicators
  const hasProductKeywords = urlLower.includes("product") || 
                            urlLower.includes("item") ||
                            urlLower.includes("goods") ||
                            urlLower.match(/\/products?\//) ||
                            urlLower.match(/\/items?\//) ||
                            urlLower.match(/\/goods?\//) ||
                            urlLower.includes("gallery") ||
                            urlLower.includes("image");
  
  // Size indicators - product images are usually larger
  const hasSizeIndicator = urlLower.match(/\d{3,4}[x×]\d{3,4}/) || // e.g., 800x600, 1920x1080
                          urlLower.includes("large") ||
                          urlLower.includes("original") ||
                          urlLower.includes("zoom") ||
                          urlLower.includes("hd") ||
                          urlLower.includes("high") ||
                          urlLower.includes("full") ||
                          urlLower.includes("big") ||
                          urlLower.match(/w_\d+|h_\d+/) || // e.g., w_800, h_600
                          urlLower.match(/\d{4,}x\d{3,}/) || // Large dimensions like 1920x1080
                          urlLower.match(/[_-](large|original|full|hd|high|big)[_-]/);
  
  // Accept if:
  // 1. Has image extension AND (product keywords OR size indicator OR is from product path)
  // 2. OR has product keywords and reasonable size (not too small)
  if (hasImageExtension) {
    if (hasProductKeywords || hasSizeIndicator || urlLower.match(/\/products?\/|\/items?\/|\/goods?\//)) {
      return true;
    }
    
    // Also accept if it's a reasonable size and doesn't have exclude patterns
    if (!hasExcludePattern && (hasSizeIndicator || urlLower.match(/\d{3,}[x×]\d{3,}/))) {
      return true;
    }
    
    // Accept images from known image hosting domains that don't have exclude patterns
    if (!hasExcludePattern && isKnownImageCDN(url)) {
      return true;
    }
    
    // Accept images from the same domain as the product page (likely product images)
    try {
      return true; // Let caller filter by domain
    } catch {
      return false;
    }
  }
  
  return false;
}

function makeAbsoluteUrl(url: string, baseUrl: string): string | null {
  if (!url) return null;

  try {
    // Already absolute URL
    if (url.startsWith("http://") || url.startsWith("https://")) {
      return url;
    }

    // Protocol-relative URL
    if (url.startsWith("//")) {
      return `https:${url}`;
    }

    // Absolute path
    if (url.startsWith("/")) {
      const urlObj = new URL(baseUrl);
      return `${urlObj.protocol}//${urlObj.host}${url}`;
    }

    // Relative URL
    const urlObj = new URL(baseUrl);
    const basePath = urlObj.pathname.substring(0, urlObj.pathname.lastIndexOf("/"));
    return `${urlObj.protocol}//${urlObj.host}${basePath}/${url}`.replace(/\/+/g, "/").replace(/:\//, "://");
  } catch (e) {
    return null;
  }
}

function extractShortDescription(html: string): string {
  // Try JSON-LD first
  try {
    const jsonLdPattern = /<script[^>]*type=["']application\/ld\+json["'][^>]*>(.*?)<\/script>/gis;
    const matches = html.matchAll(jsonLdPattern);
    for (const match of matches) {
      try {
        const jsonData = JSON.parse(match[1]);
        if (jsonData["@type"] === "Product" && jsonData.description) {
          const desc = cleanText(jsonData.description);
          if (desc && desc.length > 10) {
            return desc.substring(0, 200);
          }
        }
      } catch (e) {
        // Continue
      }
    }
  } catch (e) {
    // Continue
  }

  // Try multiple patterns
  const patterns = [
    /meta["\s]+name=["']description["'][^>]+content=["']([^"']+)["']/i,
    /og:description["\s]*content=["']([^"']+)["']/i,
    /twitter:description["\s]*content=["']([^"']+)["']/i,
    /<p[^>]*class=["'][^"']*description[^"']*["'][^>]*>(.*?)<\/p>/is,
    /<div[^>]*class=["'][^"']*short-description[^"']*["'][^>]*>(.*?)<\/div>/is,
    /product-description["\s]*>(.*?)</is,
    /<p[^>]*class=["'][^"']*product-summary[^"']*["'][^>]*>(.*?)<\/p>/is,
  ];

  for (const pattern of patterns) {
    const match = html.match(pattern);
    if (match && match[1]) {
      const text = cleanText(match[1]);
      if (text.length > 10 && text.length < 500) {
        return text.substring(0, 200);
      }
    }
  }

  return "";
}

function extractFullDescription(html: string): string {
  // Try JSON-LD first
  try {
    const jsonLdPattern = /<script[^>]*type=["']application\/ld\+json["'][^>]*>(.*?)<\/script>/gis;
    const matches = html.matchAll(jsonLdPattern);
    for (const match of matches) {
      try {
        const jsonData = JSON.parse(match[1]);
        if (jsonData["@type"] === "Product" && jsonData.description) {
          const desc = cleanText(jsonData.description);
          if (desc && desc.length > 50) {
            return desc.substring(0, 2000);
          }
        }
      } catch (e) {
        // Continue
      }
    }
  } catch (e) {
    // Continue
  }

  // Try multiple patterns for full description
  const patterns = [
    /<div[^>]*class=["'][^"']*product-description[^"']*["'][^>]*>(.*?)<\/div>/is,
    /<div[^>]*class=["'][^"']*full-description[^"']*["'][^>]*>(.*?)<\/div>/is,
    /<div[^>]*class=["'][^"']*description[^"']*full[^"']*["'][^>]*>(.*?)<\/div>/is,
    /<div[^>]*id=["']product-description["'][^>]*>(.*?)<\/div>/is,
    /<div[^>]*class=["'][^"']*description[^"']*["'][^>]*>(.*?)<\/div>/is,
    /<section[^>]*class=["'][^"']*description[^"']*["'][^>]*>(.*?)<\/section>/is,
    /product.*?description["\s]*>(.*?)</is,
    /<article[^>]*class=["'][^"']*product[^"']*["'][^>]*>(.*?)<\/article>/is,
  ];

  for (const pattern of patterns) {
    const match = html.match(pattern);
    if (match && match[1]) {
      const text = cleanText(match[1]);
      if (text.length > 50) {
        return text.substring(0, 2000);
      }
    }
  }

  // Fallback to short description
  const shortDesc = extractShortDescription(html);
  return shortDesc || "";
}

function extractProductDetails(html: string, url: string): any {
  const details: any = {};

  // Try to extract structured data (JSON-LD) - most reliable
  try {
    const jsonLdPattern = /<script[^>]*type=["']application\/ld\+json["'][^>]*>(.*?)<\/script>/gis;
    const matches = html.matchAll(jsonLdPattern);
    for (const match of matches) {
      try {
        const jsonData = JSON.parse(match[1]);
        if (jsonData["@type"] === "Product" || jsonData["@type"] === "ProductModel") {
          if (jsonData.brand) {
            details.brand = typeof jsonData.brand === "string" ? jsonData.brand : jsonData.brand.name;
          }
          if (jsonData.sku) details.sku = jsonData.sku;
          if (jsonData.mpn) details.mpn = jsonData.mpn;
          if (jsonData.gtin13 || jsonData.gtin) details.gtin = jsonData.gtin13 || jsonData.gtin;
          if (jsonData.category) details.category = jsonData.category;
          if (jsonData.description) details.description = cleanText(jsonData.description);
          
          // Extract from aggregateRating if available
          if (jsonData.aggregateRating) {
            details.rating = jsonData.aggregateRating.ratingValue;
            details.reviewCount = jsonData.aggregateRating.reviewCount;
          }

          // Extract specifications from additionalProperty
          if (jsonData.additionalProperty && Array.isArray(jsonData.additionalProperty)) {
            jsonData.additionalProperty.forEach((prop: any) => {
              if (prop.name && prop.value) {
                details[prop.name] = prop.value;
              }
            });
          }
        }
      } catch (e) {
        // Continue to next pattern
      }
    }
  } catch (e) {
    // Continue to other patterns
  }

  // Try to extract from specification tables (common in e-commerce)
  const tablePatterns = [
    /<table[^>]*class=["'][^"']*specification[^"']*["'][^>]*>(.*?)<\/table>/is,
    /<table[^>]*class=["'][^"']*specs[^"']*["'][^>]*>(.*?)<\/table>/is,
    /<table[^>]*class=["'][^"']*product-specs[^"']*["'][^>]*>(.*?)<\/table>/is,
    /<table[^>]*>(.*?)<\/table>/is,
  ];

  for (const tablePattern of tablePatterns) {
    const tableMatch = html.match(tablePattern);
    if (tableMatch) {
      const tableHtml = tableMatch[1];
      
      // Try to extract rows
      const rowPatterns = [
        /<tr[^>]*>(.*?)<\/tr>/gis,
        /<div[^>]*class=["'][^"']*spec[^"']*row[^"']*["'][^>]*>(.*?)<\/div>/gis,
      ];

      for (const rowPattern of rowPatterns) {
        const rows = tableHtml.matchAll(rowPattern);
        for (const row of rows) {
          const rowHtml = row[1];
          
          // Try different cell patterns
          const cellPatterns = [
            /<td[^>]*>(.*?)<\/td>/gis,
            /<th[^>]*>(.*?)<\/th>/gis,
            /<div[^>]*class=["'][^"']*spec-label[^"']*["'][^>]*>(.*?)<\/div>.*?<div[^>]*class=["'][^"']*spec-value[^"']*["'][^>]*>(.*?)<\/div>/gis,
          ];

          for (const cellPattern of cellPatterns) {
            const cells = Array.from(rowHtml.matchAll(cellPattern));
            if (cells.length >= 2) {
              const key = cleanText(cells[0][1] || cells[0][0]);
              const value = cleanText(cells[1][1] || cells[1][0]);
              if (key && value && key.length > 1 && value.length > 0) {
                details[key] = value;
              }
            }
          }
        }
      }
    }
  }

  // Try to extract from definition lists (dl, dt, dd)
  const dlPattern = /<dl[^>]*>(.*?)<\/dl>/gis;
  const dlMatches = html.matchAll(dlPattern);
  for (const dlMatch of dlMatches) {
    const dlHtml = dlMatch[1];
    const dtMatches = dlHtml.matchAll(/<dt[^>]*>(.*?)<\/dt>/gis);
    const ddMatches = Array.from(dlHtml.matchAll(/<dd[^>]*>(.*?)<\/dd>/gis));
    
    let index = 0;
    for (const dtMatch of dtMatches) {
      if (index < ddMatches.length) {
        const key = cleanText(dtMatch[1]);
        const value = cleanText(ddMatches[index][1]);
        if (key && value) {
          details[key] = value;
        }
        index++;
      }
    }
  }

  // Try to extract from lists with labels (common pattern)
  const listPattern = /<ul[^>]*class=["'][^"']*specification[^"']*["'][^>]*>(.*?)<\/ul>/is;
  const listMatch = html.match(listPattern);
  if (listMatch) {
    const listHtml = listMatch[1];
    const liMatches = listHtml.matchAll(/<li[^>]*>(.*?)<\/li>/gis);
    for (const liMatch of liMatches) {
      const liText = cleanText(liMatch[1]);
      const colonIndex = liText.indexOf(":");
      if (colonIndex > 0) {
        const key = liText.substring(0, colonIndex).trim();
        const value = liText.substring(colonIndex + 1).trim();
        if (key && value) {
          details[key] = value;
        }
      }
    }
  }

  // Extract from URL if it contains product info (like Microless)
  // Microless URLs contain product specs: gaming-beast-desktop-amd-ryzen-7-8700f-gigabyte-nvidia-rtx-5060-8gb-32gb-6000-mhz-ddr5-ram-1tb-m-2-pcie-gen-4-nvme-ssd-650w-psu
  try {
    const urlParts = url.split("/");
    const productSlug = urlParts[urlParts.length - 1];
    if (productSlug && productSlug.includes("-")) {
      const parts = productSlug.split("-");
      
      // Extract title from URL
      const titleWords: string[] = [];
      const specPairs: { [key: string]: string } = {};
      
      let currentKey = "";
      for (let i = 0; i < parts.length; i++) {
        const part = parts[i];
        
        // Check if it's a specification value (number with optional unit)
        if (part.match(/^\d+[a-z]*$/i) && currentKey) {
          specPairs[currentKey] = part;
          currentKey = "";
        } else if (part.match(/^(amd|intel|nvidia|ryzen|rtx|gtx|ddr|ram|ssd|hdd|psu|gb|tb|mhz|ghz|watt|w)$/i)) {
          // These are spec keywords, might be part of key or value
          if (!currentKey) {
            currentKey = part;
          } else {
            currentKey += " " + part;
          }
        } else if (part.length > 2) {
          // Potential specification key
          if (currentKey) {
            titleWords.push(currentKey);
          }
          currentKey = part;
        } else {
          titleWords.push(part);
          if (currentKey) {
            currentKey += " " + part;
          }
        }
      }
      
      // Add remaining spec pairs
      Object.assign(details, specPairs);
      
      // Try to parse common patterns from URL
      const urlText = productSlug.replace(/-/g, " ");
      
      // Extract common specs from URL text
      const specPatterns = [
        { pattern: /amd\s+ryzen\s+(\d+)/i, key: "Processor" },
        { pattern: /intel\s+core\s+(\w+\s+\d+)/i, key: "Processor" },
        { pattern: /nvidia\s+rtx\s+(\d+)/i, key: "Graphics Card" },
        { pattern: /nvidia\s+gtx\s+(\d+)/i, key: "Graphics Card" },
        { pattern: /(\d+)\s*gb\s*(?:ddr|ram)/i, key: "RAM" },
        { pattern: /(\d+)\s*tb\s*(?:ssd|hdd|storage)/i, key: "Storage" },
        { pattern: /(\d+)\s*w\s*(?:psu|power)/i, key: "Power Supply" },
        { pattern: /(\d+)\s*mhz/i, key: "RAM Speed" },
      ];
      
      specPatterns.forEach(({ pattern, key }) => {
        const match = urlText.match(pattern);
        if (match && match[1]) {
          details[key] = match[1] + (match[0].includes("gb") ? " GB" : match[0].includes("tb") ? " TB" : match[0].includes("w") ? "W" : match[0].includes("mhz") ? " MHz" : "");
        }
      });
    }
  } catch (e) {
    // Ignore URL parsing errors
  }

  return Object.keys(details).length > 0 ? details : null;
}

function extractTitleFromUrl(url: string): string {
  try {
    const urlParts = url.split("/");
    const productSlug = urlParts[urlParts.length - 1];
    if (productSlug && productSlug.includes("-")) {
      // Convert URL slug to readable title
      // e.g., "gaming-beast-desktop-amd-ryzen-7-8700f" -> "Gaming Beast Desktop AMD Ryzen 7 8700F"
      const words = productSlug.split("-");
      const titleWords: string[] = [];
      
      for (let i = 0; i < words.length; i++) {
        const word = words[i];
        const nextWord = words[i + 1];
        
        // Handle combined terms like "ryzen-7" -> "Ryzen 7"
        if (word.match(/^(ryzen|core|rtx|gtx)$/i) && nextWord && nextWord.match(/^\d+$/)) {
          titleWords.push(word.charAt(0).toUpperCase() + word.slice(1).toLowerCase() + " " + nextWord);
          i++; // Skip next word as we've combined it
          continue;
        }
        
        // Handle numbers
        if (word.match(/^\d+$/)) {
          titleWords.push(word);
          continue;
        }
        
        // Handle acronyms
        if (word.match(/^(amd|intel|nvidia|rtx|gtx|ddr|ssd|hdd|psu|gb|tb|mhz|ghz|nvme|pcie)$/i)) {
          titleWords.push(word.toUpperCase());
          continue;
        }
        
        // Regular word
        titleWords.push(word.charAt(0).toUpperCase() + word.slice(1).toLowerCase());
      }
      
      return titleWords.join(" ");
    }
  } catch (e) {
    // Ignore errors
  }
  return "";
}

function extractProductDetailsFromUrl(url: string): any {
  const details: any = {};
  try {
    const urlParts = url.split("/");
    const productSlug = urlParts[urlParts.length - 1];
    if (productSlug && productSlug.includes("-")) {
      const urlText = productSlug.replace(/-/g, " ");
      
      // Extract common specs from URL text for Microless
      const specPatterns: Array<{
        pattern: RegExp;
        key: string;
        format?: (m1: string, m2?: string) => string;
      }> = [
        { pattern: /amd\s+ryzen\s+(\d+[a-z]*)/i, key: "Processor", format: (m1) => `AMD Ryzen ${m1}` },
        { pattern: /intel\s+core\s+(\w+\s+\d+[a-z]*)/i, key: "Processor", format: (m1) => `Intel Core ${m1}` },
        { pattern: /nvidia\s+rtx\s+(\d+)/i, key: "Graphics Card", format: (m1) => `NVIDIA RTX ${m1}` },
        { pattern: /nvidia\s+gtx\s+(\d+)/i, key: "Graphics Card", format: (m1) => `NVIDIA GTX ${m1}` },
        { pattern: /gigabyte\s+nvidia\s+rtx\s+(\d+)/i, key: "Graphics Card", format: (m1) => `Gigabyte NVIDIA RTX ${m1}` },
        { pattern: /(\d+)\s*gb\s*(?:ddr|ram)/i, key: "RAM", format: (m1) => `${m1} GB` },
        { pattern: /(\d+)\s*6000\s*mhz\s*ddr5\s*ram/i, key: "RAM Speed", format: () => `6000 MHz DDR5` },
        { pattern: /ddr5\s*ram/i, key: "RAM Type", format: () => `DDR5` },
        { pattern: /(\d+)\s*tb\s*m[-\s]?2\s*pcie\s*gen\s*(\d+)\s*nvme\s*ssd/i, key: "Storage", format: (m1, m2) => `${m1} TB M.2 PCIe Gen ${m2 || "4"} NVMe SSD` },
        { pattern: /(\d+)\s*tb\s*(?:ssd|hdd|storage)/i, key: "Storage", format: (m1) => `${m1} TB` },
        { pattern: /(\d+)\s*w\s*(?:psu|power)/i, key: "Power Supply", format: (m1) => `${m1}W` },
        { pattern: /(\d+)\s*mhz/i, key: "RAM Speed", format: (m1) => `${m1} MHz` },
      ];
      
      specPatterns.forEach(({ pattern, key, format }) => {
        const match = urlText.match(pattern);
        if (match && match.length > 1) {
          if (format) {
            // Handle format function with proper arguments based on number of captures
            const args = match.slice(1);
            let value: string;
            if (args.length === 1) {
              value = format(args[0], "");
            } else if (args.length >= 2) {
              value = format(args[0], args[1]);
            } else {
              value = match[1];
            }
            details[key] = value;
          } else {
            details[key] = match[1];
          }
        }
      });
      
      // Extract brand/model if present
      if (urlText.includes("gaming beast")) {
        details["Model"] = "Gaming Beast Desktop";
      }
      if (urlText.includes("gigabyte")) {
        details["Brand"] = "Gigabyte";
      }
    }
  } catch (e) {
    // Ignore errors
  }
  return details;
}

function extractImagesAlternative(html: string, baseUrl: string): string[] {
  const images: string[] = [];
  const seen = new Set<string>();

  console.log("[Scraper] Alternative image extraction started");

  // Method 1: Look for JavaScript variables that might contain image URLs
  const jsImagePatterns = [
    /images\s*[:=]\s*\[([^\]]+)\]/gi,
    /imageUrls\s*[:=]\s*\[([^\]]+)\]/gi,
    /productImages\s*[:=]\s*\[([^\]]+)\]/gi,
    /"image"\s*:\s*"([^"]+)"/gi,
    /'image'\s*:\s*'([^']+)'/gi,
    /imageUrl\s*[:=]\s*["']([^"']+)["']/gi,
  ];

  for (const pattern of jsImagePatterns) {
    const matches = html.matchAll(pattern);
    for (const match of matches) {
      const content = match[1] || match[0];
      // Extract URLs from the content
      const urlMatches = content.matchAll(/["']([^"']+\.(jpg|jpeg|png|gif|webp|svg))["']/gi);
      for (const urlMatch of urlMatches) {
        const imgUrl = urlMatch[1];
        const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
        if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
          images.push(absoluteUrl);
          seen.add(absoluteUrl);
        }
      }
    }
  }

  // Method 2: Look for data attributes in all elements
  const dataAttrPatterns = [
    /data-image-url=["']([^"']+)["']/gi,
    /data-img-url=["']([^"']+)["']/gi,
    /data-src-large=["']([^"']+)["']/gi,
    /data-full-image=["']([^"']+)["']/gi,
    /data-zoom-url=["']([^"']+)["']/gi,
    /data-href=["']([^"']+\.(jpg|jpeg|png|gif|webp|svg))["']/gi,
  ];

  for (const pattern of dataAttrPatterns) {
    const matches = html.matchAll(pattern);
    for (const match of matches) {
      const imgUrl = match[1];
      const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
      if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
        images.push(absoluteUrl);
        seen.add(absoluteUrl);
      }
    }
  }

  // Method 3: Look for links that point to images
  const linkPatterns = [
    /<a[^>]+href=["']([^"']+\.(jpg|jpeg|png|gif|webp|svg))["']/gi,
    /href=["']([^"']+image[^"']*\.(jpg|jpeg|png|gif|webp|svg))["']/gi,
  ];

  for (const pattern of linkPatterns) {
    const matches = html.matchAll(pattern);
    for (const match of matches) {
      const imgUrl = match[1];
      const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
      if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
        images.push(absoluteUrl);
        seen.add(absoluteUrl);
      }
    }
  }

  // Method 4: Extract from inline styles with background images
  const stylePattern = /style=["'][^"']*background[^"']*url\(["']?([^"')]+)["']?\)/gi;
  const styleMatches = html.matchAll(stylePattern);
  for (const match of styleMatches) {
    const imgUrl = match[1];
    const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
    if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
      images.push(absoluteUrl);
      seen.add(absoluteUrl);
    }
  }

  // Method 5: Look for base64 images (though we'll skip these as they're usually thumbnails)
  // But we can extract regular URLs that might be in data URIs context

  // Method 6: Try to find image CDN patterns - more aggressive search
  try {
    const baseUrlObj = new URL(baseUrl);
    const baseDomain = baseUrlObj.hostname;
    const baseDomainParts = baseDomain.split('.');
    const baseDomainMain = baseDomainParts.length >= 2 
      ? baseDomainParts.slice(-2).join('.') 
      : baseDomain; // e.g., "microless.com" from "uae.microless.com"
    
    // Extract all image URLs from the HTML (more comprehensive)
    const allImageUrlPattern = /(https?:\/\/[^"'\s<>)]+\.(jpg|jpeg|png|gif|webp|svg|bmp|tiff|jfif)(\?[^"'\s<>)]*)?)/gi;
    const allMatches = Array.from(html.matchAll(allImageUrlPattern));
    
    for (const match of allMatches) {
      const imgUrl = match[1];
      if (!imgUrl) continue;
      
      try {
        const imgUrlObj = new URL(imgUrl);
        const imgDomain = imgUrlObj.hostname;
        
        // Prefer images from the same domain or known CDNs
        const isSameDomain = imgDomain === baseDomain || 
                            imgDomain.includes(baseDomainMain) ||
                            imgDomain.endsWith('.' + baseDomainMain);
        
        if (isSameDomain || isKnownImageCDN(imgUrl)) {
          const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
          if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
            images.push(absoluteUrl);
            seen.add(absoluteUrl);
          }
        }
      } catch (e) {
        // Invalid URL, skip
      }
    }
  } catch (e) {
    // Base URL is invalid, skip this method
    console.log("[Scraper] Could not parse base URL for domain matching");
  }

  // Method 7: Look for images in script tags (some sites embed image data in JS)
  const scriptPattern = /<script[^>]*>([\s\S]*?)<\/script>/gi;
  const scriptMatches = Array.from(html.matchAll(scriptPattern));
  for (const scriptMatch of scriptMatches) {
    const scriptContent = scriptMatch[1];
    // Look for image URLs in script content
    const imgUrlPattern = /(https?:\/\/[^"'\s)]+\.(jpg|jpeg|png|gif|webp|svg)(\?[^"'\s)]*)?)/gi;
    const imgMatches = Array.from(scriptContent.matchAll(imgUrlPattern));
    for (const imgMatch of imgMatches) {
      const imgUrl = imgMatch[1];
      const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
      if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
        images.push(absoluteUrl);
        seen.add(absoluteUrl);
      }
    }
  }

  // Method 8: Look for images in data attributes of any element (not just img tags)
  const allDataAttributes = Array.from(html.matchAll(/data-[^=]*=["']([^"']+\.(jpg|jpeg|png|gif|webp|svg))["']/gi));
  for (const match of allDataAttributes) {
    const imgUrl = match[1];
    const absoluteUrl = makeAbsoluteUrl(imgUrl, baseUrl);
    if (absoluteUrl && !seen.has(absoluteUrl) && isProductImage(absoluteUrl)) {
      images.push(absoluteUrl);
      seen.add(absoluteUrl);
    }
  }

  console.log(`[Scraper] Alternative extraction found ${images.length} additional images`);
  return images;
}

function isKnownImageCDN(url: string): boolean {
  const knownCDNs = [
    "cloudinary.com",
    "imgix.net",
    "images.unsplash.com",
    "cdn.shopify.com",
    "amazonaws.com",
    "googleusercontent.com",
    "cloudflare.com",
    "cdn.", // Any CDN subdomain
    "static.", // Static content
    "media.", // Media content
    "images.", // Images subdomain
    "img.", // Img subdomain
    "pic.", // Pic subdomain
  ];
  return knownCDNs.some((cdn) => url.toLowerCase().includes(cdn));
}

function cleanText(text: string): string {
  if (!text) return "";
  return text
    .replace(/<[^>]+>/g, " ") // Remove HTML tags
    .replace(/&nbsp;/g, " ")
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&apos;/g, "'")
    .replace(/&#x27;/g, "'")
    .replace(/&#x2F;/g, "/")
    .replace(/\s+/g, " ")
    .trim();
}