Initial commit

Author: Roshrini

Readme.txt

@@ -0,0 +1,15 @@
+Instructions to run our project on cade:
+
+1) We are using Python3.5 and NLTK package
+2) Please run following command and install additional packages.
+/usr/local/stow/python/amd64_linux26/python-3.5.0/bin/python3 -m nltk.downloader -d /home/$USERNAME/nltk_data all
+
+Replace $USERNAME with your cade username.
+
+Open our folder. Go to question-answers folder where there are other files.
+
+3) /usr/local/stow/python/amd64_linux26/python-3.5.0/bin/python3 question_answers.py ./developset/input.txt > Response.answer
+
+After running above command, Response.answer file will be generated. This is our output file.
+
+You can calculate accuracy on this file. We got accuracy as 25.89% on developset.

contributors.txt

@@ -0,0 +1,239 @@
+import nltk
+
+from nltk.tokenize import word_tokenize
+from nltk.tokenize import sent_tokenize
+from nltk.corpus import stopwords
+import string
+from nltk.stem.lancaster import LancasterStemmer
+
+
+name_list = []
+location_list = []
+month_list = []
+time_list = []
+occupation_list = []
+
+stopwordSet = stopwords.words('english')
+morePunctuations = set(['``','"','...',"''","n't","'re","'s","--"])
+punctuationSet = set(string.punctuation) | morePunctuations
+lancaster_stemmer = LancasterStemmer()
+
+
+def parse_story(story_filename):
+     story_dict = {}
+     with open(story_filename) as myfile:
+         parts = myfile.read().split("TEXT:")
+
+         headline = parts[0].splitlines()[0]
+         date = parts[0].splitlines()[1]
+         storyid = parts[0].splitlines()[2]
+         text = sent_tokenize(parts[1].lstrip("\n").replace("\n"," "))
+         story_dict[(headline,date,storyid)] = text
+     return story_dict
+
+
+def removeStopWordsAndTagPOS(story_dict):
+    storyWithoutStopWords_dict = {}
+    storyPOS_dict = {}
+    for key in story_dict:
+        text = story_dict[key]
+        for line in text:
+            words = word_tokenize(line)
+            lineWithoutStopWord = []
+            for word in words:
+                if word.lower() not in stopwordSet:
+                    if word.lower() not in punctuationSet:
+                        lineWithoutStopWord.append(word)
+            storyWithoutStopWords_dict[line] = lineWithoutStopWord
+            storyPOS_dict[line] = nltk.pos_tag(lineWithoutStopWord)
+
+    return storyWithoutStopWords_dict, storyPOS_dict
+
+def camel(s):
+    return (s != s.lower() and s != s.upper())
+
+def contains_proper_noun(question):
+    proper_noun = ""
+    wordsInAQuestion = word_tokenize(question)
+    questionWithoutStopWord = []
+    for word in wordsInAQuestion:
+        if word.lower() not in stopwordSet:
+            if word.lower() not in punctuationSet:
+                questionWithoutStopWord.append(word)
+
+    for word in questionWithoutStopWord:
+        if (camel(word)):
+            proper_noun = proper_noun +" "+ word
+
+    proper_noun_list = proper_noun.split()
+    for each_proper_noun in proper_noun_list:
+        if any(each_proper_noun in s for s in name_list):
+            return True
+        else:
+            return False
+
+
+def semantic_classes(name_filename):
+    with open(name_filename+"names.txt") as f:
+        name_list.append(f.read().splitlines())
+   # print(type(name_list))
+
+    with open(name_filename+"location.txt") as f:
+        location_list.append(f.read().splitlines())
+
+    with open(name_filename+"month.txt") as f:
+        month_list.append(f.read().lower().splitlines())
+
+    with open(name_filename+"time.txt") as f:
+        time_list.append(f.read().lower().splitlines())
+
+    with open(name_filename+"occupation.txt") as f:
+        occupation_list.append(f.read().lower().splitlines())
+#    print(occupation_list)
+
+
+def contains_name_word(sent):
+    proper_noun = ""
+    wordsInASent = word_tokenize(sent)
+    sentWithoutStopWord = []
+    for word in wordsInASent:
+                if word.lower() not in stopwordSet:
+                    if word.lower() not in punctuationSet:
+                        sentWithoutStopWord.append(word)
+
+    if any("name" in s for s in sentWithoutStopWord):
+        return True
+    else:
+        return False
+
+def contains_name_occupation(sent):
+    proper_noun = ""
+    wordsInASent = word_tokenize(sent)
+    sentWithoutStopWord = []
+    for word in wordsInASent:
+        if word.lower() not in stopwordSet:
+            if word.lower() not in punctuationSet:
+                sentWithoutStopWord.append(word)
+
+    for word in sentWithoutStopWord:
+        if (camel(word)):
+            proper_noun = proper_noun +" "+ word
+
+    proper_noun_list = proper_noun.split()
+
+    for each_proper_noun in proper_noun_list:
+        if any(each_proper_noun in s for s in name_list):
+            return True
+
+    for word in sentWithoutStopWord:
+        if any(word in s for s in occupation_list):
+         #   print(word)
+            return True
+    return False
+
+
+def who_rule(question, sent, storyPOS_dict):
+    score = 0
+    status = False
+    score = score+ wordMatch(question,sent,storyPOS_dict)
+    if(not contains_proper_noun(question) and contains_proper_noun(sent)):
+        score = score + 6
+    if (not contains_proper_noun(question) and contains_name_word(sent)):
+        score = score + 4
+    status = contains_name_occupation(sent)
+    if (status):
+        score = score + 4
+  #  print(score)
+
+def when_rule(question, sent):
+    print()
+
+def data_forward(questions_data,story_dict):
+    storyWithoutStopWords_dict,storyPOS_dict = removeStopWordsAndTagPOS(story_dict)
+
+    for question in questions_data:
+        for story_key in story_dict:
+            text_list = story_dict[story_key]
+            wordMatch(question[1],text_list,storyPOS_dict)
+            for sent in text_list:
+                who_rule(question[1],sent,storyPOS_dict)
+
+def wordMatch(question, text, storyPOS_dict):
+    wordsInAQuestion = word_tokenize(question)
+    rootsInAQuestion = set()
+    for word in wordsInAQuestion:
+        root = lancaster_stemmer.stem(word)
+        rootsInAQuestion.add(root)
+
+    for line in storyPOS_dict:
+        verbmatch_score = 0
+        rootmatch_score = 0
+        scoreOfALine = {}
+        for (word,tag) in storyPOS_dict[line]:
+            if 'V' in tag:
+                verb_root = lancaster_stemmer.stem(word)
+                if verb_root in rootsInAQuestion:
+                    verbmatch_score = verbmatch_score + 6
+            else:
+                word_root = lancaster_stemmer.stem(word)
+                if word_root in rootsInAQuestion:
+                    rootmatch_score = rootmatch_score + 3
+        scoreOfALine[line] = rootmatch_score + verbmatch_score
+#         print(scoreOfALine)
+# print("\n")
+
+
+def main():
+    input_path = "/Users/roshaninagmote/Downloads/sample/"
+    input_file = open(input_path+"/input.txt")
+    semantic_classes("/Users/roshaninagmote/PycharmProjects/question-answers/")
+
+
+    input_data = input_file.read().splitlines()
+    path = input_data[0]
+
+    for i in range(1,len(input_data)):
+
+        each_story = input_data[i]+".story"
+        each_question = input_data[i]+".questions"
+        story_file = open(input_path+each_story)
+        questions_file = open(input_path+each_question)
+        story_data = story_file.read()
+        questions_data_raw = questions_file.read().splitlines()
+        questions_total = filter(None, questions_data_raw)
+
+        que = questions_file.read()
+
+        questions_data = []
+        for j in range(0,len(questions_total),3):
+            question_temp = []
+            quesid = questions_total[j].split(":")[1].lstrip(" ")
+            question_temp.append(quesid)
+            ques = questions_total[j+1].split(":")[1].lstrip(" ")
+            question_temp.append(ques)
+            question_temp.append(questions_total[j+2])
+
+            questions_data.append(question_temp)
+
+
+        story_dict = parse_story(input_path+each_story)
+     #   print(story_dict)
+        data_forward(questions_data,story_dict)
+    print("\n")
+
+if __name__ == "__main__":
+    main()
+
+
+ if not any(word in question[1].lower() for word in quest_words):
+                print("roshani")
+                max_score_else = 0
+                for sent in text_list:
+                    current_score = wordMatch(question[1],sent,storyPOS_dict)
+                    if current_score > max_score_else:
+                        max_score_else = current_score
+                        answer = sent
+               # print "in else", question[1], sent
+                print "QuestionID:",question[0]
+                print "Answer:", answer
+        print("\n")

developset/1999-W02-5.answers

@@ -0,0 +1,30 @@
+QuestionID: 1999-W02-5-1
+Question: Where is South Queens Junior High School located?
+Answer: Liverpool, Nova Scotia | Canada
+Difficulty: Moderate
+
+QuestionID: 1999-W02-5-2
+Question: Who is the principal of South Queens Junior High School?
+Answer: Betty Jean Aucoin
+Difficulty: Easy
+
+QuestionID: 1999-W02-5-3
+Question: What has South Queens Junior High School done with its old metal shop?
+Answer: turned it into a fitness club
+Difficulty: Moderate
+
+QuestionID: 1999-W02-5-5
+Question: Who runs the club?
+Answer: a non-profit society | school and community volunteers
+Difficulty: Moderate
+
+QuestionID: 1999-W02-5-6
+Question: How big is the club?
+Answer: 12,000 square feet
+Difficulty: Easy
+
+QuestionID: 1999-W02-5-7
+Question: If you were a student, how much would a club membership cost you?
+Answer: $135 a year
+Difficulty: Moderate 
+

developset/1999-W02-5.questions

@@ -0,0 +1,24 @@
+QuestionID: 1999-W02-5-1
+Question: Where is South Queens Junior High School located?
+Difficulty: Moderate
+
+QuestionID: 1999-W02-5-2
+Question: Who is the principal of South Queens Junior High School?
+Difficulty: Easy
+
+QuestionID: 1999-W02-5-3
+Question: What has South Queens Junior High School done with its old metal shop?
+Difficulty: Moderate
+
+QuestionID: 1999-W02-5-5
+Question: Who runs the club?
+Difficulty: Moderate
+
+QuestionID: 1999-W02-5-6
+Question: How big is the club?
+Difficulty: Easy
+
+QuestionID: 1999-W02-5-7
+Question: If you were a student, how much would a club membership cost you?
+Difficulty: Moderate 
+

developset/1999-W02-5.story

@@ -0,0 +1,51 @@
+HEADLINE: Nova Scotia School Pumps Up
+DATE: January 8, 1999
+STORYID: 1999-W02-5
+
+TEXT:
+
+A middle school in Liverpool, Nova Scotia is pumping up bodies as well
+as minds.
+
+It's an example of a school teaming up with the community to raise
+money. South Queens Junior High School is taking aim at the fitness
+market.
+
+The school has turned its one-time metal shop - lost to budget cuts
+almost two years ago - into a money-making professional fitness club.
+The club will be open seven days a week.
+
+The club, operated by a non-profit society made up of school and
+community volunteers, has sold more than 30 memberships and hired a
+full-time co-ordinator.
+
+Principal Betty Jean Aucoin says the club is a first for a Nova Scotia
+public school. She says the school took it on itself to provide a
+service needed in Liverpool.
+
+"We don't have any athletic facilities here on the South Shore of Nova
+Scotia, so if we don't use our schools, communities such as Queens are
+going to be struggling to get anything going," Aucoin said.
+
+More than a $100,000 was raised through fund-raising and donations from
+government, Sport Nova Scotia, and two local companies.
+
+Some people are wondering if the ties between the businesses and the
+school are too close. Schools are not set up to make profits or promote
+businesses.  
+
+Southwest Regional School Board superintendent Ann Jones says there's no
+fear the lines between education and business are blurring.
+
+"First call on any school facility belongs to... the youngsters in the
+school," says Ann Jones.
+
+The 12,000-square-foot club has seven aerobic machines, including
+treadmills, steppers, and stationary bicycles, as well as weight
+machines and freeweights.
+
+Memberships cost $180 a year for adults and $135 for students and
+seniors.
+
+Proceeds pay the salary of the centre co-ordinator and upkeep of the
+facility.