MultiThreadedSearchEngine/cleanhref_test.go at main · Amurr1/MultiThreadedSearchEngine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
package main

import (
	"flag"
	"testing"
)

func TestCleanHref(t *testing.T) {
	tests := []struct {
		name       string
		cur_url    string
		href       string
		clean_want string
	}{
		{
			name:       "home -> simple",
			cur_url:    "https://usf-cs272-f25.github.io/test-data/project01/",
			href:       "/test-data/project01/simple.html",
			clean_want: "https://usf-cs272-f25.github.io/test-data/project01/simple.html",
		},
		{
			name:       "home -> style",
			cur_url:    "https://usf-cs272-f25.github.io/test-data/project01/",
			href:       "/test-data/project01/href.html",
			clean_want: "https://usf-cs272-f25.github.io/test-data/project01/href.html",
		},
		{
			name:       "home -> style",
			cur_url:    "https://usf-cs272-f25.github.io/test-data/project01/",
			href:       "/test-data/project01/style.html",
			clean_want: "https://usf-cs272-f25.github.io/test-data/project01/style.html",
		},
		{
			name:       "href -> simple",
			cur_url:    "https://usf-cs272-f25.github.io/test-data/project01/href.html",
			href:       "/test-data/project01/simple.html",
			clean_want: "https://usf-cs272-f25.github.io/test-data/project01/simple.html",
		},
		{
			name:       "style -> simple",
			cur_url:    "https://usf-cs272-f25.github.io/test-data/project01/style.html",
			href:       "/test-data/project01/simple.html",
			clean_want: "https://usf-cs272-f25.github.io/test-data/project01/simple.html",
		},
		{
			name:       "style -> style",
			cur_url:    "https://usf-cs272-f25.github.io/test-data/project01/style.html",
			href:       "/test-data/project01/href.html",
			clean_want: "https://usf-cs272-f25.github.io/test-data/project01/href.html",
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			// Create stop words list
			stop_words, err := CreateStopWordSet("stopwords-en.json")
			if err != nil {
				t.Fatal("Could not create stop words list")
				return
			}

			// Parse flags to get data_handler type
			flag.Parse()
			args := flag.Args()
			var data_handler DataHandler
			if len(args) >= 2 && args[0] == "ondisk" {
				var err error
				data_handler, err = NewSQLDataBase(args[1])
				if err != nil {
					data_handler = NewInvIdx()
				}
			} else {
				data_handler = NewInvIdx()
			}
			t.Cleanup(func() {
				data_handler.DeleteDataBase()
			})

			// Create crawler and run test
			crawler := NewCrawler(data_handler, stop_words, 0.5)
			clean_got, err := crawler.CleanHref(tc.cur_url, tc.href)
			if err != nil {
				t.Fatalf("Clean of %s from %s failed.\nwant: %s", tc.href, tc.cur_url, tc.clean_want)
			}
			if clean_got != tc.clean_want {
				t.Fatalf("Clean of %s from %s gave:\n%s\nwant: %s", tc.href, tc.cur_url, clean_got, tc.clean_want)
			}
		})
	}
}