MultiThreadedSearchEngine/search_test.go at main · Amurr1/MultiThreadedSearchEngine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
package main

import (
	"flag"
	"reflect"
	"testing"
)

func TestSearch(t *testing.T) {
	tests := []struct {
		word      string
		start_url string
		map_want  map[string]uint
	}{
		{
			"Verona",
			"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.0.html",
			map[string]uint{"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.0.html": 1},
		},
		{
			"Benvolio",
			"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.1.html",
			map[string]uint{"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.1.html": 26},
		},
		{
			"Romeo",
			"https://usf-cs272-f25.github.io/test-data/rnj/",
			map[string]uint{
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.0.html":  2,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.1.html":  22,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.3.html":  2,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.4.html":  17,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.5.html":  15,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneII_30.2.html": 42,
				"https://usf-cs272-f25.github.io/test-data/rnj/":                  200,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneI_30.2.html":  15,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneII_30.0.html": 3,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneII_30.1.html": 10,
				"https://usf-cs272-f25.github.io/test-data/rnj/sceneII_30.3.html": 13,
			},
		},
	}

	for _, tc := range tests {
		t.Run(tc.word, func(t *testing.T) {
			// Create stop words list
			stop_words, err := CreateStopWordSet("stopwords-en.json")
			if err != nil {
				t.Fatal("Could not create stop words list")
				return
			}

			// Parse flags to get data_handler type
			flag.Parse()
			args := flag.Args()
			var data_handler DataHandler
			if len(args) >= 2 && args[0] == "ondisk" {
				var err error
				data_handler, err = NewSQLDataBase(args[1])
				if err != nil {
					data_handler = NewInvIdx()
				}
			} else {
				data_handler = NewInvIdx()
			}
			t.Cleanup(func() {
				data_handler.DeleteDataBase()
			})

			// Create search engine
			se, err := NewSearchEngine(
				data_handler,
				stop_words,
				0.001,
				struct {
					clean_href int
					download   int
					extract    int
				}{-1, -1, -1},
				tc.start_url,
				false,
			)
			if err != nil {
				t.Fatal("Crawler creation had issue opening stopwords file.")
			}

			// Do the search
			cleaned_word := NormalizeWords([]string{tc.word})[0]
			raw_map_got, err := se.Search(cleaned_word)
			map_got := make(map[string]uint)
			for k, v := range raw_map_got {
				map_got[k] = v.freq
			}

			// Check against expected results
			if err != nil {
				t.Fatalf("Search(%q, %q) had error searching for word", tc.start_url, tc.word)
			}
			if !reflect.DeepEqual(map_got, tc.map_want) {
				t.Errorf("Search(%q, %q) got:\n%v\nwant:\n%v", tc.start_url, tc.word, MapToString(map_got), MapToString(tc.map_want))
			}
		})
	}
}