Cloned

Shubh-Yadav/Automatic-Indian-Sign-Language-Translator

README.md

@@ -1,2 +1,15 @@
-# Automatic-Indian-Sign-Language-ISLTranslator-
-I created an application which takes in live speech or audio recording as input, converts it into text and displays the relevant Indian Sign Language images or GIFs, using Natural Language Processing and Machine Learning Algorithm.
+# Speech-to-Sign-language-Translator
+**An application which takes in live speech or audio recording as input, converts it into text and displays the relevant Indian Sign Language images or GIFs.**
+- Front-end using EasyGui.
+- Speech as input through microphone using PyAudio. 
+- Speech recognition using Google Speech API.
+- Text Preprocessing using NLP.
+- Dictionary based Machine Translation.
+
+## To run the application.
+1. Open the Downloads folder and then open the terminal.
+2. From the terminal, run the *main* python file using the command **python main.py**.
+3. The application interface appears on the screen.
+4. Hit the record button to start taking speech as input.
+5. Any speech recorded is then processed and respective outputs are shown accordingly.
+6. To exit the application using speech, say *goodbye*.

main.py

@@ -0,0 +1,133 @@
+import speech_recognition as sr
+import numpy as np
+import matplotlib.pyplot as plt
+import cv2
+from easygui import *
+import os
+from PIL import Image, ImageTk
+from itertools import count
+import tkinter as tk
+import string
+#import selecting
+# obtain audio from the microphone
+def func():
+        r = sr.Recognizer()
+        isl_gif=['all the best', 'any questions', 'are you angry', 'are you busy', 'are you hungry', 'are you sick', 'be careful',
+                'can we meet tomorrow', 'did you book tickets', 'did you finish homework', 'do you go to office', 'do you have money',
+                'do you want something to drink', 'do you want tea or coffee', 'do you watch TV', 'dont worry', 'flower is beautiful',
+                'good afternoon', 'good evening', 'good morning', 'good night', 'good question', 'had your lunch', 'happy journey',
+                'hello what is your name', 'how many people are there in your family', 'i am a clerk', 'i am bore doing nothing', 
+                 'i am fine', 'i am sorry', 'i am thinking', 'i am tired', 'i dont understand anything', 'i go to a theatre', 'i love to shop',
+                'i had to say something but i forgot', 'i have headache', 'i like pink colour', 'i live in nagpur', 'lets go for lunch', 'my mother is a homemaker',
+                'my name is john', 'nice to meet you', 'no smoking please', 'open the door', 'please call an ambulance', 'please call me later',
+                'please clean the room', 'please give me your pen', 'please use dustbin dont throw garbage', 'please wait for sometime', 'shall I help you',
+                'shall we go together tommorow', 'sign language interpreter', 'sit down', 'stand up', 'take care', 'there was traffic jam', 'wait I am thinking',
+                'what are you doing', 'what is the problem', 'what is todays date', 'what is your age', 'what is your father do', 'what is your job',
+                'what is your mobile number', 'what is your name', 'whats up', 'when is your interview', 'when we will go', 'where do you stay',
+                'where is the bathroom', 'where is the police station', 'you are wrong','address','agra','ahemdabad', 'all', 'april', 'assam', 'august', 'australia', 'badoda', 'banana', 'banaras', 'banglore',
+'bihar','bihar','bridge','cat', 'chandigarh', 'chennai', 'christmas', 'church', 'clinic', 'coconut', 'crocodile','dasara',
+'deaf', 'december', 'deer', 'delhi', 'dollar', 'duck', 'febuary', 'friday', 'fruits', 'glass', 'grapes', 'gujrat', 'hello',
+'hindu', 'hyderabad', 'india', 'january', 'jesus', 'job', 'july', 'july', 'karnataka', 'kerala', 'krishna', 'litre', 'mango',
+'may', 'mile', 'monday', 'mumbai', 'museum', 'muslim', 'nagpur', 'october', 'orange', 'pakistan', 'pass', 'police station',
+'post office', 'pune', 'punjab', 'rajasthan', 'ram', 'restaurant', 'saturday', 'september', 'shop', 'sleep', 'southafrica',
+'story', 'sunday', 'tamil nadu', 'temperature', 'temple', 'thursday', 'toilet', 'tomato', 'town', 'tuesday', 'usa', 'village',
+'voice', 'wednesday', 'weight']
+
+
+        arr=['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r',
+        's','t','u','v','w','x','y','z']
+        with sr.Microphone() as source:
+
+                r.adjust_for_ambient_noise(source) 
+                i=0
+                while True:
+                        print('Say something')
+                        audio = r.listen(source)
+
+                                                        # recognize speech using Sphinx
+                        try:
+                                a=r.recognize_google(audio)
+                                print("you said " + a.lower())
+
+                                for c in string.punctuation:
+                                    a= a.replace(c,"")
+
+                                if(a.lower()=='goodbye'):
+                                        print("oops!Time To say good bye")
+                                        break
+
+                                elif(a.lower() in isl_gif):
+
+                                    class ImageLabel(tk.Label):
+                                            """a label that displays images, and plays them if they are gifs"""
+                                            def load(self, im):
+                                                if isinstance(im, str):
+                                                    im = Image.open(im)
+                                                self.loc = 0
+                                                self.frames = []
+
+                                                try:
+                                                    for i in count(1):
+                                                        self.frames.append(ImageTk.PhotoImage(im.copy()))
+                                                        im.seek(i)
+                                                except EOFError:
+                                                    pass
+
+                                                try:
+                                                    self.delay = im.info['duration']
+                                                except:
+                                                    self.delay = 100
+
+                                                if len(self.frames) == 1:
+                                                    self.config(image=self.frames[0])
+                                                else:
+                                                    self.next_frame()
+
+                                            def unload(self):
+                                                self.config(image=None)
+                                                self.frames = None
+
+                                            def next_frame(self):
+                                                if self.frames:
+                                                    self.loc += 1
+                                                    self.loc %= len(self.frames)
+                                                    self.config(image=self.frames[self.loc])
+                                                    self.after(self.delay, self.next_frame)
+
+                                    root = tk.Tk()
+                                    lbl = ImageLabel(root)
+                                    lbl.pack()
+                                    lbl.load(r'/home/shubh/Videos/ISL_CODE/ISL/ISL_Gifs/{0}.gif'.format(a.lower()))
+                                    root.mainloop()
+                                else:
+
+                                    for i in range(len(a)):
+                                                    #a[i]=a[i].lower()
+                                                    if(a[i] in arr):
+
+                                                            ImageAddress = 'letters/'+a[i]+'.jpg'
+                                                            ImageItself = Image.open(ImageAddress)
+                                                            ImageNumpyFormat = np.asarray(ImageItself)
+                                                            plt.imshow(ImageNumpyFormat)
+                                                            plt.draw()
+                                                            plt.pause(0.8) # pause how many seconds
+                                                            #plt.close()
+                                                    else:
+                                                            continue
+
+                        except:
+                               print("Could not listen")
+                        plt.close()
+
+#func()
+while 1:
+  image   = "signlang.png"
+  msg="HEARING IMPAIRMENT ASSISTANT"
+  choices = ["Live Voice","All Done!","Recorded Audio"]
+  reply   = buttonbox(msg,image=image,choices=choices)
+  if reply ==choices[0]:
+        func()
+  if reply == choices[1]:
+        quit()
+  if reply==choices[2]:
+        os.system("selecting.py")