crawl_delay_test.go

package main

import (
	"flag"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"
	"time"
)

func TestCrawlDelay(t *testing.T) {
	tests := []struct {
		name  string
		delay float32
	}{
		{name: "delay_0.01", delay: 0.01},
		{name: "delay_0.02", delay: 0.02},
		{name: "delay_0.05", delay: 0.05},
		{name: "delay_0.1", delay: 0.1},
		{name: "delay_0.2", delay: 0.2},
	}

	// Start file server
	fs := http.FileServer(http.Dir("./SearchEngine"))
	ts := httptest.NewServer(http.StripPrefix("/", fs))
	defer ts.Close()

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			// Create stop words list
			stop_words, err := CreateStopWordSet("stopwords-en.json")
			if err != nil {
				t.Fatal("Could not create stop words list")
				return
			}

			// Parse flags to get data_handler type
			flag.Parse()
			args := flag.Args()
			var data_handler DataHandler
			if len(args) >= 2 && args[0] == "ondisk" {
				var err error
				data_handler, err = NewSQLDataBase(args[1])
				if err != nil {
					data_handler = NewInvIdx()
				}
			} else {
				data_handler = NewInvIdx()
			}
			t.Cleanup(func() {
				data_handler.DeleteDataBase()
			})

			// Write robots.txt to allow only index pages and set Crawl-delay
			robots_text := fmt.Sprintf(
				`
				User-agent: *
				Disallow: /
				Allow: *index.html
				Crawl-delay: %v
				`,
				tc.delay,
			)
			robots_path := filepath.Join("SearchEngine", "robots.txt")
			if err := os.WriteFile(robots_path, []byte(robots_text), 0644); err != nil {
				t.Fatalf("could not write robots.txt: %v", err)
			}
			t.Cleanup(func() {
				_ = os.Remove(robots_path)
			})

			// Time crawl
			crawler := NewCrawler(data_handler, stop_words, 0.5)
			start := time.Now()
			startURL := ts.URL + "/top10/index.html"
			_, err = crawler.Crawl(
				startURL,
				struct {
					clean_href int
					download   int
					extract    int
				}{16, 16, 16},
			)
			if err != nil {
				t.Fatalf("Crawl failed: %v", err)
			}
			elapsed := time.Since(start)

			// Find document size
			size, err := data_handler.Size()
			if err != nil {
				t.Fatalf("Data handler Size failed: %v", err)
			}
			if size == 0 {
				t.Fatalf("no documents crawled; unexpected test setup")
			}

			// Check duration expectation (its delay between crawls, so thats why it's size - 1)
			expected := time.Duration(float64(size-1) * float64(tc.delay) * float64(time.Second))
			if elapsed < expected {
				t.Fatalf("crawl too fast: elapsed=%v want>=%v (docs=%d delay=%v)", elapsed, expected, size, tc.delay)
			}
		})
	}
}