commit 0fe375cfaad63a8820067fea1ff292701224d643 Author: dev6tools Date: Sun Aug 23 15:04:56 2026 -0400 Initial release: stealth browser (Camoufox + captcha WebUI) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b3260dd --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +data/ +__pycache__/ +*.log +.env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f064045 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11-slim + +# Playwright/Firefox runtime deps (GTK, X11, fonts, dbus) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgtk-3-0 libdbus-glib-1-2 libx11-xcb1 libxt6 libasound2 \ + libxcomposite1 libxdamage1 libxrandr2 libgbm1 libpango-1.0-0 \ + libcairo2 libnss3 libxss1 fonts-liberation ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Pre-download the Camoufox browser so first request is fast +RUN python -m camoufox fetch + +COPY app.py . +COPY static/ static/ + +EXPOSE 8000 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..5fd5342 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# Stealth Browser (Camoufox) for pi + +Bypasses bot detection / fingerprint blocking that the built-in `web_scrape` +tool hits (Cloudflare, DataDome, 403s, captcha walls). Includes a **manual +CAPTCHA-solving WebUI** and **admin log panel**. + +**Deployed:** http://192.168.0.65:8010 (login: admin / admin — change it) +**Health:** http://192.168.0.65:8010/health + +## Architecture + +``` +pi (any machine) --HTTP--> stealth-browser container (192.168.0.65:8010) + web_fetch_stealth tool Camoufox patched Firefox, headless + │ + ├─ captcha? → WebUI challenge (login admin/admin) + └─ logs → /api/logs (admin panel tab) +``` + +pi extension `~/.pi/agent/extensions/stealth-tools.ts` registers +`web_fetch_stealth`, pointing at the server by default. Override with +`STEALTH_BROWSER_URL`, add basic auth with `STEALTH_BROWSER_USER` / +`STEALTH_BROWSER_PASS` (server uses `ADMIN_USER` / `ADMIN_PASS`). + +## WebUI (admin + captcha solving) + +- **Challenges tab** — live queue; when a fetch hits a captcha the job pauses + and appears with ⚠. Click *Open & solve*: live screenshot, click-to-interact, + type into focused field, reload/back/scroll, goto URL, then **✔ Done — + extract page** (or ✖ cancel). +- **Logs tab** — every fetch: start/captcha/action/done/failed/cancelled with + status, duration, error. Stored in `./data/logs.jsonl`. + +## Redeploy after changes (on Docker host, 192.168.0.65) + +```bash +cd ~/stealth-browser # synced from /blackbox/.../stealth-browser +docker compose up -d --build +curl localhost:8010/health +``` + +## Proxy (important) + +Fingerprint evasion alone gets blocked on datacenter IPs. For Cloudflare/ +high-security sites, set a rotating residential proxy: + +```yaml +environment: + - PROXY=http://user:pass@residential-proxy.example:port +``` + +## API + +- `GET /health` — liveness +- `POST /fetch` — `{"url": "...", "extract": "text|html", "selector": "css?", "pause": bool}` + Blocks until done (or captcha solved via WebUI). Returns + `{status, title, url, text, outcome, job_id}`. +- `GET /api/jobs` — job queue +- `GET /api/jobs/{id}` — job + result text +- `GET /api/jobs/{id}/screenshot` — live JPEG +- `POST /api/jobs/{id}/action` — `click|x,y|img_w`, `type`, `key`, `scroll`, `goto`, `reload`, `back` +- `POST /api/jobs/{id}/continue|cancel` +- `GET /api/logs` — last 200 log entries + +## Limits + +- Defeats fingerprint bot detection. Does NOT defeat: + - IP reputation blocking (needs the residential proxy above) + - CAPTCHAs it can't flag — detection looks for known captcha iframes + (Cloudflare, hCaptcha, reCAPTCHA, Turnstile, Arkose); pass `"pause": true` + to always pause for manual review. +- `MAX_CONCURRENT=3` browser instances; each fetch boots a fresh browser (~2-5s). + +## Deploy notes / gotchas fixed + +- Port 8000 was taken by Nextcloud → service runs on host port **8010**. +- Camoufox needs `headless=True` explicitly (defaults to headful → DISPLAY error). +- Docker blocks Firefox's user-namespace sandbox → `MOZ_DISABLE_*_SANDBOX=1` env. +- GTK/X11/font system packages required in image. \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..4847777 --- /dev/null +++ b/app.py @@ -0,0 +1,335 @@ +import asyncio +import json +import os +import secrets +import time +import uuid + +from fastapi import Depends, FastAPI, HTTPException, Request +from fastapi.responses import FileResponse, JSONResponse, Response +from fastapi.security import HTTPBasic, HTTPBasicCredentials +from pydantic import BaseModel +from playwright.async_api import TimeoutError as PlaywrightTimeoutError + +from camoufox.async_api import AsyncCamoufox + +app = FastAPI(title="Stealth Browser API", version="1.1") + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +DATA_DIR = os.getenv("DATA_DIR", "/data") +LOG_FILE = os.path.join(DATA_DIR, "logs.jsonl") + +PROXY = os.getenv("PROXY", "") +ADMIN_USER = os.getenv("ADMIN_USER", "admin") +ADMIN_PASS = os.getenv("ADMIN_PASS", "") # empty = no auth (LAN mode) +CAPTCHA_WAIT_MS = int(os.getenv("CAPTCHA_WAIT_MS", "4000")) +SOLVE_TIMEOUT_S = int(os.getenv("SOLVE_TIMEOUT_S", "900")) + +CAPTCHA_SELECTORS = ( + 'iframe[src*="challenges.cloudflare.com"],' + 'iframe[src*="hcaptcha.com"],' + 'iframe[src*="recaptcha"],' + 'iframe[src*="arkoselabs"],' + 'iframe[src*="funcaptcha"],' + 'iframe[src*="turnstile"],' + 'div[style*="captcha"]' +) +MAX_CONCURRENT = int(os.getenv("MAX_CONCURRENT", "3")) +_semaphore = asyncio.Semaphore(MAX_CONCURRENT) + +# ---------------------------------------------------------------- auth +security = HTTPBasic(auto_error=False) + + +def require_auth(creds: HTTPBasicCredentials | None = Depends(security)): + if not ADMIN_PASS: + return None # open mode + if not creds or not secrets.compare_digest(creds.username, ADMIN_USER) or not secrets.compare_digest(creds.password, ADMIN_PASS): + raise HTTPException(401, "Invalid or missing credentials") + return creds + + +# ---------------------------------------------------------------- browser helper +def browser_kwargs() -> dict: + kwargs = {"headless": True} + if PROXY: + kwargs["proxy"] = {"server": PROXY} + return kwargs + + +# ---------------------------------------------------------------- logging +_log_lock = asyncio.Lock() + + +async def log(entry: dict): + os.makedirs(DATA_DIR, exist_ok=True) + entry["ts"] = time.strftime("%Y-%m-%d %H:%M:%S") + async with _log_lock: + with open(LOG_FILE, "a") as f: + f.write(json.dumps(entry) + "\n") + + +async def read_logs(limit: int = 200): + if not os.path.exists(LOG_FILE): + return [] + lines = [] + async with _log_lock: + with open(LOG_FILE) as f: + lines = f.readlines() + out = [] + for line in lines[-limit:]: + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + pass + return list(reversed(out)) + + +# ---------------------------------------------------------------- jobs +class Job: + def __init__(self, url: str, extract: str, selector: str | None, pause: bool): + self.id = uuid.uuid4().hex[:8] + self.url = url + self.extract = extract + self.selector = selector + self.pause = pause + self.created = time.time() + self.status = "queued" # queued|loading|captcha|solving|done|failed|cancelled|timeout + self.outcome = None + self.error = None + self.title = None + self.status_code = None + self.final_url = None + self.text = None + self.viewport = {"width": 1280, "height": 720} + self.page = None + self.solved = asyncio.Event() + self.cancelled = False + self.action_lock = asyncio.Lock() + self.finished = asyncio.Event() + + def to_dict(self): + return { + "id": self.id, "url": self.url, "status": self.status, + "outcome": self.outcome, "created": self.created, + "title": self.title, "status_code": self.status_code, + "error": self.error, + } + + +jobs: dict[str, Job] = {} + + +class FetchRequest(BaseModel): + url: str + extract: str = "text" # text | html + selector: str | None = None + pause: bool = False # always open admin WebUI for manual check + + +# ---------------------------------------------------------------- fetch flow +async def detect_captcha(page) -> bool: + try: + await page.wait_for_selector(CAPTCHA_SELECTORS, state="attached", timeout=CAPTCHA_WAIT_MS) + return True + except PlaywrightTimeoutError: + return False + except Exception: + return False + + +async def run_fetch(job: Job): + t0 = time.time() + async with _semaphore: + try: + async with AsyncCamoufox(**browser_kwargs()) as browser: + page = await browser.new_page() + job.page = page + job.status = "loading" + await log({"job": job.id, "url": job.url, "event": "load_start"}) + resp = await page.goto(job.url, wait_until="domcontentloaded", timeout=45_000) + job.status_code = resp.status if resp else None + job.final_url = page.url + job.title = await page.title() + + has_captcha = job.pause or await detect_captcha(page) + if has_captcha and not job.cancelled: + job.status = "captcha" + await log({"job": job.id, "url": job.url, "event": "captcha_detected"}) + try: + await asyncio.wait_for(job.solved.wait(), timeout=SOLVE_TIMEOUT_S) + except asyncio.TimeoutError: + job.status = "timeout" + job.outcome = "captcha_timeout" + job.error = "No human solved the captcha in time" + await log({"job": job.id, "url": job.url, "event": "captcha_timeout"}) + return + if job.cancelled: + return + + if job.cancelled: + return + job.status = "solving" + try: + await page.wait_for_load_state("networkidle", timeout=15_000) + except Exception: + pass + job.title = await page.title() + if job.selector: + el = page.locator(job.selector) + job.text = await el.first.inner_text() if await el.count() else "" + elif job.extract == "html": + job.text = await page.content() + else: + job.text = await page.inner_text("body") + job.status = "done" + job.outcome = "captcha_solved" if has_captcha else "ok" + await log({ + "job": job.id, "url": job.url, "event": "done", + "outcome": job.outcome, "status_code": job.status_code, + "duration_ms": int((time.time() - t0) * 1000), "title": job.title, + }) + except Exception as e: + job.status = "failed" + job.error = str(e) + job.outcome = "failed" + await log({ + "job": job.id, "url": job.url, "event": "failed", + "error": str(e)[:500], "duration_ms": int((time.time() - t0) * 1000), + }) + finally: + job.page = None + if job.outcome == "cancelled": + await log({"job": job.id, "url": job.url, "event": "cancelled", "duration_ms": int((time.time() - t0) * 1000)}) + job.finished.set() + + +@app.post("/fetch", dependencies=[Depends(require_auth)]) +async def fetch(req: FetchRequest): + if not req.url.startswith(("http://", "https://")): + raise HTTPException(400, "url must start with http(s)://") + job = Job(req.url, req.extract, req.selector, req.pause) + jobs[job.id] = job + asyncio.create_task(run_fetch(job)) + await job.finished.wait() + if job.status in ("done",): + return {"status": job.status_code, "title": job.title, "url": job.final_url, "text": (job.text or "")[:200_000], "outcome": job.outcome, "job_id": job.id} + raise HTTPException(502, f"{job.outcome or job.status}: {job.error or 'unknown error'}") + + +# ---------------------------------------------------------------- admin / WebUI endpoints +class Action(BaseModel): + type: str # click|type|key|scroll|goto|reload|back + x: float | None = None + y: float | None = None + img_w: float | None = None + text: str | None = None + key: str | None = None + delta: int | None = None + url: str | None = None + + +@app.get("/", include_in_schema=False) +async def index(): + return FileResponse(os.path.join(BASE_DIR, "static", "index.html")) + + +@app.get("/health") +async def health(): + return {"ok": True, "version": app.version, "active": sum(1 for j in jobs.values() if j.status in ("loading", "captcha", "solving"))} + + +@app.get("/api/jobs", dependencies=[Depends(require_auth)]) +async def list_jobs(): + return [j.to_dict() for j in jobs.values()][-50:][::-1] + + +@app.get("/api/jobs/{job_id}", dependencies=[Depends(require_auth)]) +async def job_detail(job_id: str): + job = jobs.get(job_id) + if not job: + raise HTTPException(404, "job not found") + d = job.to_dict() + d["text"] = (job.text or "")[:50_000] + return d + + +@app.get("/api/jobs/{job_id}/screenshot", dependencies=[Depends(require_auth)]) +async def screenshot(job_id: str): + job = jobs.get(job_id) + if not job or job.page is None: + raise HTTPException(404, "no live page for this job") + png = await job.page.screenshot(type="jpeg", quality=60) + return Response(content=png, media_type="image/jpeg") + + +async def run_action(job: Job, action: Action): + page = job.page + if page is None: + raise HTTPException(404, "page closed") + a = action + if a.type == "click": + vw, vh = job.viewport["width"], job.viewport["height"] + img_w = a.img_w or vw + x = a.x * vw / img_w + y = a.y * vh / img_w # same scale + await page.mouse.click(x, y) + elif a.type == "type": + await page.keyboard.type(a.text or "", delay=25) + elif a.type == "key": + await page.keyboard.press(a.key or "Enter") + elif a.type == "scroll": + await page.mouse.wheel(0, a.delta or 300) + elif a.type == "goto": + await page.goto(a.url, wait_until="domcontentloaded", timeout=45_000) + elif a.type == "reload": + await page.reload(wait_until="domcontentloaded") + elif a.type == "back": + await page.go_back(wait_until="domcontentloaded") + else: + raise HTTPException(400, f"unknown action {a.type}") + await log({"job": job.id, "event": "action", "type": a.type}) + return job.to_dict() + + +@app.post("/api/jobs/{job_id}/action", dependencies=[Depends(require_auth)]) +async def do_action(job_id: str, action: Action): + job = jobs.get(job_id) + if not job: + raise HTTPException(404) + async with job.action_lock: + return await run_action(job, action) + + +@app.post("/api/jobs/{job_id}/continue", dependencies=[Depends(require_auth)]) +async def continue_job(job_id: str): + job = jobs.get(job_id) + if not job: + raise HTTPException(404) + job.status = "solving" + job.solved.set() + return {"ok": True} + + +@app.post("/api/jobs/{job_id}/cancel", dependencies=[Depends(require_auth)]) +async def cancel_job(job_id: str): + job = jobs.get(job_id) + if not job: + raise HTTPException(404) + job.cancelled = True + job.outcome = "cancelled" + job.status = "cancelled" + job.solved.set() + return {"ok": True} + + +@app.get("/api/logs", dependencies=[Depends(require_auth)]) +async def get_logs(): + return await read_logs() + + +# periodic log of health to admin logs +@app.on_event("startup") +async def startup(): + await log({"event": "server_start", "version": app.version, "proxy": bool(PROXY)}) + print(f"[startup] auth={'on' if ADMIN_PASS else 'OFF (LAN open)'} captcha_wait={CAPTCHA_WAIT_MS}ms solve_timeout={SOLVE_TIMEOUT_S}s", flush=True) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6f5f96e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +services: + stealth-browser: + build: . + container_name: stealth-browser + ports: + - "8010:8000" + environment: + # Optional rotating residential proxy. Without it, datacenter IPs still + # get blocked by Cloudflare/RiskIQ. Format: http://user:pass@host:port + - PROXY= + # Admin WebUI login (change these!). Empty ADMIN_PASS = no login (LAN open). + - ADMIN_USER=admin + - ADMIN_PASS=admin + - CAPTCHA_WAIT_MS=4000 # how long to look for a captcha after load + - SOLVE_TIMEOUT_S=900 # how long a challenge stays open before timing out + - MAX_CONCURRENT=3 + # Docker can't give firefox its user-namespace sandbox + - MOZ_DISABLE_CONTENT_SANDBOX=1 + - MOZ_DISABLE_GMP_SANDBOX=1 + - MOZ_DISABLE_RDD_SANDBOX=1 + volumes: + - ./data:/data # admin logs (logs.jsonl) + restart: unless-stopped + # Keep browser memory reasonable and allow many tabs + shm_size: "1gb" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1343708 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +camoufox[geoip]>=0.4.15 +fastapi>=0.115 +uvicorn[standard]>=0.30 \ No newline at end of file diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..2aed813 --- /dev/null +++ b/static/index.html @@ -0,0 +1,227 @@ + + + + + +Stealth Browser Admin + + + +

🛡️ Stealth Browser Admin —

+ +
+ + +
+ +
+
+
+
+ +
+ +
TimeJobEventURL / DetailDur
loading…
+
+ +
+
+
+ + + + + + + + + + + + +
+
+
+ +
+ +
+

Login

+ + + +
Wrong credentials
+
Stored in this browser only. Set ADMIN_USER/ADMIN_PASS on the server to require login.
+
+ + + + \ No newline at end of file