MultiThreadedSearchEngine/extract_test.go at main · Amurr1/MultiThreadedSearchEngine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
package main

import (
	"flag"
	"testing"
)

func TestExtract(t *testing.T) {
	tests := []struct {
		name           string
		html_data      []byte
		words_want     map[string]uint
		hrefs_want     map[string]struct{}
		tot_words_want uint
	}{
		{
			name: "/test-data/project01/",
			html_data: []byte(`
			       <html>
			       <body>
				       <ul>
					       <li>
					       <a href="/test-data/project01/simple.html">simple.html</a>
					       </li>
					       <li>
					       <a href="/test-data/project01/href.html">href.html</a>
					       </li>
					       <li>
					       <a href="/test-data/project01/style.html">style.html</a>
				       </ul>
			       </body>
			       </html>
		       `),
			words_want: map[string]uint{"simpl": 1, "href": 1, "style": 1},
			hrefs_want: map[string]struct{}{
				"/test-data/project01/simple.html": {},
				"/test-data/project01/href.html":   {},
				"/test-data/project01/style.html":  {},
			},
			tot_words_want: 6,
		},
		{
			name: "/test-data/project01/simple.html",
			html_data: []byte(`
			       <html>
			       <body>
				       Hello CS 272, there are no links here.
			       </body>
			       </html>
		       `),
			words_want:     map[string]uint{"link": 1},
			hrefs_want:     map[string]struct{}{},
			tot_words_want: 7,
		},
		{
			name: "/test-data/project01/href.html",
			html_data: []byte(`
			       <html>
			       <body>
				       For a simple example, see <a href="/test-data/project01/simple.html">simple.html</a>
			       </body>
			       </html>
		       `),
			words_want:     map[string]uint{"simpl": 2},
			hrefs_want:     map[string]struct{}{"/test-data/project01/simple.html": {}},
			tot_words_want: 7,
		},
		{
			name: "/test-data/project01/style.html",
			html_data: []byte(`
			       <html>
			       <head>
				       <title>Style</title>
				       <style>
					       a.blue {
					       color: blue;
					       }
					       a.red {
					       color: red;
					       }
				       </style>
			       <body>
				       <p>
					       Here is a blue link to <a class="blue" href="/test-data/project01/href.html">href.html</a>
				       </p>
				       <p>
					       And a red link to <a class="red" href="/test-data/project01/simple.html">simple.html</a>
				       </p>
			       </body>
			       </html>
		       `),
			words_want: map[string]uint{"style": 1, "blue": 1, "link": 2, "href": 1, "red": 1, "simpl": 1},
			hrefs_want: map[string]struct{}{
				"/test-data/project01/simple.html": {},
				"/test-data/project01/href.html":   {},
			},
			tot_words_want: 16,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			// Create stop words list
			stop_words, err := CreateStopWordSet("stopwords-en.json")
			if err != nil {
				t.Fatal("Could not create stop words list")
				return
			}

			// Parse flags to get data_handler type
			flag.Parse()
			args := flag.Args()
			var data_handler DataHandler
			if len(args) >= 2 && args[0] == "ondisk" {
				var err error
				data_handler, err = NewSQLDataBase(args[1])
				if err != nil {
					data_handler = NewInvIdx()
				}
			} else {
				data_handler = NewInvIdx()
			}
			t.Cleanup(func() {
				data_handler.DeleteDataBase()
			})

			// Create crawler and extract
			crawler := NewCrawler(data_handler, stop_words, 0.5)
			_, words_got, hrefs_got, tot_words_got := crawler.Extract(tc.html_data)

			// Check words map
			if len(words_got) != len(tc.words_want) {
				t.Fatalf("Extract from %s: got %d words, want %d", tc.name, len(words_got), len(tc.words_want))
			}
			for word, wantCount := range tc.words_want {
				gotCount, ok := words_got[word]
				if !ok || gotCount != wantCount {
					t.Fatalf("Extract from %s: word '%s' got %d, want %d", tc.name, word, gotCount, wantCount)
				}
			}
			// Check hrefs map
			if len(hrefs_got) != len(tc.hrefs_want) {
				t.Fatalf("Extract from %s: got %d hrefs, want %d", tc.name, len(hrefs_got), len(tc.hrefs_want))
			}
			for href := range tc.hrefs_want {
				if _, ok := hrefs_got[href]; !ok {
					t.Fatalf("Extract from %s: missing href '%s'", tc.name, href)
				}
			}
			// Check total words
			if tc.tot_words_want != tot_words_got {
				t.Fatalf("Extract from %s got: %d words; want: %d", tc.name, tot_words_got, tc.tot_words_want)
			}
		})
	}
}