disallow_test.go

package main

import (
	"flag"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

func TestDisallow(t *testing.T) {
	tests := []struct {
		name        string
		global_want bool
		chap21_want bool
		robots_text string
	}{
		{
			name:        "allow_all",
			global_want: true,
			chap21_want: true,
			robots_text: `
				User-agent: *
				Disallow: 
			`,
		},
		{
			name:        "disallow_all",
			global_want: false,
			chap21_want: false,
			robots_text: `
				User-agent: *
				Disallow: /
			`,
		},
		{
			name:        "allow_chap21_only",
			global_want: false,
			chap21_want: true,
			robots_text: `
				User-agent: *
				Disallow: /
				Allow: */The%20Project%20Gutenberg%20eBook%20of%20The%20Prince,%20by%20Nicolo%20Machiavelli/index.html
				Allow: *chap21.html
			`,
		},
		{
			name:        "disallow_chap21_only",
			global_want: true,
			chap21_want: false,
			robots_text: `
				User-agent: *
				Disallow: *chap21.html
			`,
		},
		{
			name:        "allow_over_disallow_chap21",
			global_want: true,
			chap21_want: true,
			robots_text: `
				User-agent: *
				Disallow: *chap21.html
				Allow: *chap21.html
			`,
		},
	}

	// Start file server
	fs := http.FileServer(http.Dir("./SearchEngine"))
	ts := httptest.NewServer(http.StripPrefix("/", fs))
	defer ts.Close()

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			// Create stop words list
			stop_words, err := CreateStopWordSet("stopwords-en.json")
			if err != nil {
				t.Fatal("Could not create stop words list")
				return
			}

			// Parse flags to get data_handler type
			flag.Parse()
			args := flag.Args()
			var data_handler DataHandler
			if len(args) >= 2 && args[0] == "ondisk" {
				var err error
				data_handler, err = NewSQLDataBase(args[1])
				if err != nil {
					data_handler = NewInvIdx()
				}
			} else {
				data_handler = NewInvIdx()
			}
			t.Cleanup(func() {
				data_handler.DeleteDataBase()
			})

			// Create robots.txt
			robots_path := filepath.Join("SearchEngine", "robots.txt")
			if err := os.WriteFile(robots_path, []byte(tc.robots_text), 0644); err != nil {
				t.Fatalf("could not write root robots.txt: %v", err)
			}
			t.Cleanup(func() {
				_ = os.Remove(robots_path)
			})

			// Start crawl and get hrefs
			crawler := NewCrawler(data_handler, stop_words, 0.001)
			startURL := ts.URL + "/top10/index.html"
			_, err = crawler.Crawl(
				startURL,
				struct {
					clean_href int
					download   int
					extract    int
				}{16, 16, 16},
			)
			if err != nil {
				t.Fatalf("Crawl failed: %v", err)
			}
			hrefs, err := data_handler.GetAllDocs()
			if err != nil {
				t.Fatalf("Data handler GetAllWordCount failed: %v", err)
			}

			// Try to find something thats not chap21 or a path to it (ie chap22)
			global_got := false
			for href := range hrefs {
				if strings.HasSuffix(href, "chap22.html") {
					global_got = true
					break
				}
			}

			// Try to find chap21
			chap21_got := false
			for href := range hrefs {
				if strings.HasSuffix(href, "chap21.html") {
					chap21_got = true
					break
				}
			}

			// Check against desired outputs
			if global_got != tc.global_want {
				t.Fatalf("test %s: global crawled want=%v, got=%v", tc.name, tc.global_want, global_got)
			}
			if chap21_got != tc.chap21_want {
				t.Fatalf("test %s: chap21 crawled want=%v, got=%v", tc.name, tc.chap21_want, chap21_got)
			}
		})
	}
}