From 8915314aa449deea2137e845efef851e0e07f7f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=BD=95=E6=AD=A2?=
 <68958533+h3zh1@users.noreply.github.com>
Date: Sat, 27 Jun 2026 02:46:03 +0800
Subject: [PATCH] fix: make crawl honor max body length

---
 core/e2e_test.go      | 40 ++++++++++++++++++++++++++++++++++++++++
 core/run_with_args.go |  2 +-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/core/e2e_test.go b/core/e2e_test.go
index a0c03cc..952c46e 100644
--- a/core/e2e_test.go
+++ b/core/e2e_test.go
@@ -483,6 +483,46 @@ func TestE2E_CrawlRecursive(t *testing.T) {
 	}
 }
 
+func TestE2E_CrawlHonorsMaxLength(t *testing.T) {
+	var visited sync.Map
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		visited.Store(r.URL.Path, true)
+		w.Header().Set("Content-Type", "text/html")
+		switch r.URL.Path {
+		case "/":
+			w.WriteHeader(200)
+			fmt.Fprint(w, `<html><head><title>Index Page</title></head><body>`)
+			fmt.Fprint(w, strings.Repeat("padding ", 200))
+			fmt.Fprint(w, `<a href="/after-limit">late link</a></body></html>`)
+		case "/after-limit":
+			w.WriteHeader(200)
+			fmt.Fprint(w, `<html><body>this page should not be crawled when max-length is 1KB</body></html>`)
+		default:
+			w.WriteHeader(404)
+			fmt.Fprint(w, "not found")
+		}
+	}))
+	defer server.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := runSpray(t, ctx, []string{
+		"-u", server.URL,
+		"--crawl",
+		"--crawl-depth", "1",
+		"--max-length", "1",
+		"--no-bar", "-q", "--no-stat",
+	})
+	if err != nil {
+		t.Fatalf("RunWithArgs with --crawl: %v", err)
+	}
+
+	if _, ok := visited.Load("/after-limit"); ok {
+		t.Fatal("crawl followed a link located beyond --max-length 1KB")
+	}
+}
+
 func TestE2E_CrawlRelativeAssetsUnderBasePath(t *testing.T) {
 	var visited sync.Map
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
diff --git a/core/run_with_args.go b/core/run_with_args.go
index 1212a37..9a1a629 100644
--- a/core/run_with_args.go
+++ b/core/run_with_args.go
@@ -157,7 +157,7 @@ func RunWithArgs(ctx context.Context, args []string, opts RunOptions) error {
 		return err
 	}
 	defer runner.CloseFiles()
-	if option.ReadAll || runner.CrawlPlugin {
+	if option.ReadAll {
 		ihttp.DefaultMaxBodySize = -1
 	}