modified scraper

Author: sayan0506

dockerfile

@@ -25,6 +25,7 @@ WORKDIR /app
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PORT=8080
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 #ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 # ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
 # ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright

requirements.txt

@@ -10,7 +10,7 @@ langchain_core==0.3.66
 #langchain_google_vertexai==2.0.26
 langgraph==0.5.0
 Pillow==11.2.1
-playwright==1.50.0
+#playwright==1.50.0
 protobuf==3.20.3
 python-dotenv==1.1.1
 streamlit==1.46.0

scraper.py

@@ -52,6 +52,9 @@ async def scrape_content(self, state: WorkflowState) -> WorkflowState:
             browser = await p.chromium.launch(headless=True,
                                               args = [
                                                 "--no-sandbox",
+                                                "--disable-setuid-sandbox",
+                                                "--no-zygote",
+                                                "--single-process",
                                                 "--disable-dev-shm-usage",
                                                 "--disable-gpu"
                                               ])
@@ -108,7 +111,12 @@ async def scrape_content(self, state: WorkflowState) -> WorkflowState:
 
             except Exception as e:
                 print(f"Error scraping content: {e}")
-                return None
+                #return None
+                return {
+                    **state,
+                    "status": "scrape_failed",
+                    "error": str(e)
+                }
 
             finally:
                 # close the browser