import requests
import json
import os

# Replace with your actual Firecrawl API Key
FIRECRAWL_API_KEY = "YOUR_FIRECRAWL_API_KEY"
FIRECRAWL_API_URL = "https://api.firecrawl.dev/v0/scrape"

def scrape_website_firecrawl(url):
    """
    Scrapes a given URL using the Firecrawl API.

    Args:
        url (str): The URL to scrape.

    Returns:
        str: The scraped content in Markdown format, or None if an error occurs.
    """
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {FIRECRAWL_API_KEY}"
    }
    payload = {
        "url": url,
        "pageOptions": {
            "onlyMainContent": True  # Focus on the main content of the page
        }
    }

    try:
        response = requests.post(FIRECRAWL_API_URL, headers=headers, json=payload)
        response.raise_for_status()  # Raise an exception for bad status codes (4xx or 5xx)
        data = response.json()
        if data and data.get("success") and data.get("data") and data["data"][0].get("content"):
            return data["data"][0]["content"]
        else:
            print(f"Error: Firecrawl did not return expected data for {url}")
            return None
    except requests.exceptions.RequestException as e:
        print(f"Error scraping {url}: {e}")
        return None

if __name__ == "__main__":
    target_url = "https://blog.firecrawl.dev/blog/scraping-dynamic-content-with-firecrawl-and-playwright/" # Example URL
    scraped_content = scrape_website_firecrawl(target_url)

    if scraped_content:
        print("Successfully scraped content (first 500 characters):")
        print(scraped_content[:500])
        # You might want to save this to a file for larger content
        with open("scraped_content.md", "w", encoding="utf-8") as f:
            f.write(scraped_content)
        print("\nScraped content saved to scraped_content.md")
    else:
        print("Failed to scrape content.")

import requests
import json
import os

# --- (Previous Firecrawl scraping code goes here) ---

# Replace with your actual Firecrawl API Key
FIRECRAWL_API_KEY = "YOUR_FIRECRAWL_API_KEY"
FIRECRAWL_API_URL = "https://api.firecrawl.dev/v0/scrape"

def scrape_website_firecrawl(url):
    """
    Scrapes a given URL using the Firecrawl API.

    Args:
        url (str): The URL to scrape.

    Returns:
        str: The scraped content in Markdown format, or None if an error occurs.
    """
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {FIRECRAWL_API_KEY}"
    }
    payload = {
        "url": url,
        "pageOptions": {
            "onlyMainContent": True  # Focus on the main content of the page
        }
    }

    try:
        response = requests.post(FIRECRAWL_API_URL, headers=headers, json=payload)
        response.raise_for_status()  # Raise an exception for bad status codes (4xx or 5xx)
        data = response.json()
        if data and data.get("success") and data.get("data") and data["data"][0].get("content"):
            return data["data"][0]["content"]
        else:
            print(f"Error: Firecrawl did not return expected data for {url}")
            return None
    except requests.exceptions.RequestException as e:
        print(f"Error scraping {url}: {e}")
        return None


# --- LLM Integration ---

# Placeholder for your LLM API details
LLM_API_URL = "https://api.your-llm-provider.com/v1/generate" # Example URL
LLM_API_KEY = "YOUR_LLM_API_KEY" # Replace with your LLM API Key

def send_to_llm(text_content, prompt):
    """
    Sends the text content and a prompt to a hypothetical LLM API.

    Args:
        text_content (str): The scraped text content to send to the LLM.
        prompt (str): The prompt to instruct the LLM.

    Returns:
        str: The LLM's response, or None if an error occurs.
    """
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {LLM_API_KEY}" # Or whatever auth your LLM uses
    }
    
    # The payload structure will vary greatly depending on your LLM provider.
    # This is a common structure for text generation.
    payload = {
        "model": "your-preferred-llm-model", # e.g., "gpt-4", "gemini-1.5-pro", etc.
        "messages": [
            {"role": "system", "content": "You are a helpful assistant that can summarize and analyze documents."},
            {"role": "user", "content": f"{prompt}\n\nHere is the document:\n\n{text_content}"}
        ],
        "max_tokens": 1000, # Adjust as needed
        "temperature": 0.7 # Adjust for creativity vs. factualness
    }

    try:
        response = requests.post(LLM_API_URL, headers=headers, json=payload)
        response.raise_for_status()
        llm_data = response.json()
        # This part also depends on the LLM's response structure
        if llm_data and llm_data.get("choices") and llm_data["choices"][0].get("message") and llm_data["choices"][0]["message"].get("content"):
            return llm_data["choices"][0]["message"]["content"]
        else:
            print("Error: LLM did not return expected response structure.")
            return None
    except requests.exceptions.RequestException as e:
        print(f"Error communicating with LLM API: {e}")
        return None

if __name__ == "__main__":
    target_url = "https://blog.firecrawl.dev/blog/scraping-dynamic-content-with-firecrawl-and-playwright/" # Example URL
    scraped_content = scrape_website_firecrawl(target_url)

    if scraped_content:
        print("\n--- Sending scraped content to LLM ---")
        user_prompt = "Summarize the key points of this article in under 200 words."
        llm_response = send_to_llm(scraped_content, user_prompt)

        if llm_response:
            print("\nLLM Response:")
            print(llm_response)
        else:
            print("Failed to get response from LLM.")
    else:
        print("Skipping LLM interaction due to failed scraping.")