From e9b922e8ec167a720dc5cec43a8fb36a96796ec7 Mon Sep 17 00:00:00 2001 From: abhijithneilabraham Date: Sat, 29 Aug 2020 07:43:59 +0530 Subject: [PATCH 1/2] clause classification using cqa --- tableqa/clauses.py | 34 ++++++++++++---------------------- tableqa/nlp.py | 34 ++++------------------------------ tableqa/values.json | 2 +- tableqa/vocab.json | 2 +- 4 files changed, 18 insertions(+), 54 deletions(-) diff --git a/tableqa/clauses.py b/tableqa/clauses.py index b32d2e5..d59f7fc 100755 --- a/tableqa/clauses.py +++ b/tableqa/clauses.py @@ -1,28 +1,18 @@ - -from tensorflow.keras.models import load_model -from sentence_transformers import SentenceTransformer -from numpy import asarray - +from nlp import qa +import tensorflow as tf class Clause: def __init__(self): - self.bert_model = SentenceTransformer('bert-base-nli-mean-tokens') - self.model=load_model("Question_Classifier.h5") - self.types={0:'SELECT {} FROM {}', 1:'SELECT MAX({}) FROM {}', 2:'SELECT MIN({}) FROM {}', 3:'SELECT COUNT({}) FROM {}', 4:'SELECT SUM({}) FROM {}', 5:'SELECT AVG({}) FROM {}'} - - def adapt(self,q,inttype=False,priority=False): - emb=asarray(self.bert_model.encode(q)) - self.clause=self.types[self.model.predict_classes(emb)[0]] - if priority and inttype and "COUNT" in self.clause: - self.clause= '''SELECT SUM({}) FROM {}''' - return self.clause - - - - - - + self.base_q="what is {} here" + self.types={"the entity":'SELECT {} FROM {}', "the maximum":'SELECT MAX({}) FROM {}', "the minimum":'SELECT MIN({}) FROM {}', "counted":'SELECT COUNT({}) FROM {}', "summed":'SELECT SUM({}) FROM {}', "averaged":'SELECT AVG({}) FROM {}'} + def adapt(self,q,inttype=False,priority=False): + scores={} + validated_scores={} + total_scores={} + for k,v in self.types.items(): + scores[k]=tf.convert_to_tensor(qa(q,self.base_q.format(k),return_score=True)[1]) + return self.types[max(scores, key=scores.get)] - + \ No newline at end of file diff --git a/tableqa/nlp.py b/tableqa/nlp.py index 91cfdf7..7ed2d76 100644 --- a/tableqa/nlp.py +++ b/tableqa/nlp.py @@ -13,8 +13,6 @@ qa_model = TFBertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad') qa_tokenizer = BertTokenizer.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad') -rerank_model = TFAutoModelForSequenceClassification.from_pretrained('bert-large-cased-whole-word-masking') -rerank_tokenizer = AutoTokenizer.from_pretrained('bert-large-cased-whole-word-masking') @@ -165,6 +163,9 @@ def _find(lst, sublst): def _window_overlap(s1, e1, s2, e2): return s2 <= e1 if s1 neg_logit or not filter_cols: - scores.append(score - neg_logit) - return scores - + def cond_map(self,s): data=conditions diff --git a/tableqa/values.json b/tableqa/values.json index df74dbe..242045a 100644 --- a/tableqa/values.json +++ b/tableqa/values.json @@ -1 +1 @@ -{"Activity_Type_Chapter": ["Perdiem - Short Stay", "Anesthesia", "Perdiem - SCBU - Daily Rate (Day 4 to 8)", "Perdiem - New-born Nursery - Daily Rate (Day 4 to 8)", "Auditory System", "Bleaching Vital", "Room and Board: First Class Room", "Sensory organs", "Newborns & Other Neonates", "Temporary Procedures/Professional Services", "General", "Perdiem - ICU/CCU - Daily Rate (Day 15 to 21)", "Case Presentation/ Treatment Planning", "Room Rate difference - Daily Rate (Day 1 and more ) - Suite", "Restoration", "Room Rate difference - Daily Rate (Day 1 and more ) - Royal Suite", "Perdiem - NICU - Daily Rate (Day 8 to 14)", "Operating Room Services - Minor Surgery", "Perdiem - Long Term Stay - Daily Rate - Intensive Cases", "Periodontal Services, Miscellaneous", "Dentures, Partial, Cast With Acrylic Base", "Nuclear Medicine", "Maxillofacial Deformities, Treatment Of", "Restorations, Foil, Gold", "Pontics, Bridge", "Drugs Other Than Chemotherapy", "Consultation - Specialist", "Educational services rendered by Clinician (Physician or Non-Physician) to patient opting out of the Comprehensive Screening, new or established patient; 10-15 minutes.", "Vision Services", "Other Medical Services", "Room Rate difference - Daily Rate (Day 1 and more ) - Isolation Room", "Operating Room Services - First Hour", "Breast Mammography", "Laboratory And Expense Procedures (This Code Is Used In Conjunction With The \"+ L\" And \"+ E\" Designation Following Specific Codes In The Guide. The Addition Of These Codes Are To Facilitate Computer Or Manual Input For Third Party Claims Processing, Perso", "Diseases & Disorders Of The Hepatobiliary System & Pancreas", "Perdiem - ICU/CCU - Daily Rate (Day 22 and more)", "Removals, (Extractions), Erupted Teeth", "Medical Ambulatory Visits", "Diseases & Disorders Of The Urinary Tract", "Diseases & Disorders Of The Respiratory System", "Anaesthesia", "Maternity Care & Delivery", "Musculo-skeletal system", "Temporary Codes assigned to DME Regional Carriers", "Appliances", "Perdiem - SCU - Daily Rate (Day 9 and more)", "Recontouring Of Retainer/Pontics,\n(Of Existing Bridgework)", "Per Diem- Bundled Base Payment of home care (Level 2- Intermediate ) * -A bundled payment that includes all medical services and transportation - Su", "Professional Services", "Neonatal Intensive Care Unit (NICU)", "Mixed Dentition", "Appliances, Active, For Tooth Guidance Or Minor Tooth Movement", "Factors Influencing Health Status & Other Contacts With Health Services", "Perdiem - NICU - Daily Rate (Day 15 to 21)", "Repairs", "Pulp Chamber, Treatment Of, (Excluding Final Restoration)", "Diseases & Disorders Of The Myeloproliferative System & Poorly Differentiated Neoplasms", "Root Canal Therapy\nTo Include: Treatment Plan, Clinical Procedures (I.E. Pulpectomy, Biomechanical Preparation, Chemotherapeutic Treatment And Obturation), With Appropriate Radiographs And Excluding Final Restoration.", "Per Diem- Bundled Base Payment of home care (Level 3- Intensive) * -A bundled payment that includes all medical services and tra", "Perdiem - Day Stay (Day care) - Inclusive.", "Perdiem - Ward or Shared Room - Daily Rate (Day 8 and more)", "Catheterization Lab", "Polishing", "Radiation Oncology", "Perdiem - Day Stay (Day care) Surgical Case - Inclusive.", "Musculoskeletal", "Restorations, Inlays, Onlays, Pins And Posts", "Nervous System", "Dentures, Partial, Acrylic", "Periodontal Services, Surgical (Includes Local Anaesthetic, Suturing And The Placement And Removal Of Initial Surgical Dressing. A Surgical Site Is An Area That Lends Itself To One Or More Procedures. It Is Considered To Include A Full Quadrant, Sextant O", "Fixed Prosthetics, Other Services", "Day Stay (Day care) Room - Daily Rate", "Non-surgical cleansing of a wound without debridement, with or without local anesthesia, with or without the application of a surgical dressing: 16 sq inches / 100 sq centimeters or less.", "Dental Procedures", "Operating Room Services - Every Additional 1/2 hr.", "Coronary Care Unit (CCU)", "Dentures, Partial (Fabricated In Conjunction With A Denturist)", "Respiratory Durable Medical Equipment, Inexpensive and Routinely Purchased", "Frequent Preventative Services Billed On A Time Unit Basis. Services Covered Under This Code Are: Polishing, Scaling, Root Planing, Nutritional Counselling, Oral Hygiene Instruction/Plaque Control And Application Of An Antimicrobial Agent\n***Provinces Usi", "Temporary Codes Assigned by CMS", "Endocrine System", "Radiographs (Including Radiographic Examination And Diagnosis And Interpretation)", "Perdiem - Ward or Shared Room - Daily Rate (Day 4 to 8)", "Cardiovascular System", "Perdiem - PICU - Daily Rate (Day 22 and more)", "Perdiem - New-born Nursery - Daily Rate (Day 1 to 3)", "Restorations, Amalgam", "Perdiem - SCBU - Daily Rate (Day 1 to 3)", "Diseases & Disorders Of The Ear, Nose, Mouth & Throat", "Perdiem - Continuous Ambulatory Peritoneal Dialysis (CAPD).", "Urinary System", "Emergency Room - Hourly Rate", "Antiinfectives for systemic use", "Remodelling And Recontouring Oral Tissues In Preparation For Removable Prostheses (To Include Codes 73110, 73120, 73140, 73150, 73160, 73170, 73180)", "Periodontal Services, Non Surgical", "Endodontic, Procedures, Miscellaneous", "Prosthetics", "Preventive Services, Other", "Infectious & Parasitic Diseases Of Systemic Or Unspecified Sites", "Perdiem - SCU - Daily Rate (Day 1 to 3)", "Paediatric intensive care Unit (PICU)", "Drugs/Medication, Dispensing", "Non-surgical cleansing of a wound without debridement, with or without local anesthesia, with or without the application of a surgical dressing: between 16 sq inches / 100 sq centimeters and 48 sq inches / 300 sq centimeters .", "DRG Add-on Payment for High Cost Consumables", "Eye & Ocular Adnexa", "Dentures, Replication, Relining And Rebasing", "Room Rate difference - Daily Rate (Day 1 and more ) - VIP Room", "Mediastinum & Diaphragm", "Prosthondics, Removable, Fabricated In Conjunction With A Denturist", "Special Care Baby Unit (SCBU)", "Periodontal Procedures, Adjunctive (When Per Joint Is Designated, The Corresponding Tooth Code Is Represented By The Mesial Of The Tooth Involved, Except At The Midline, Where The Tooth To The Right Of The Joint Is Utilized.)", "Respiratory System", "Room and Board: VIP Room", "Various", "Tele-consultation resulting in Referral", "Perdiem - ICU/CCU - Daily Rate (Day 8 to 14)", "Intensive Care Unit (ICU)", "Integumentary", "Outlier Payment", "Durable Medical Equipment", "Female Genital System", "Evaluation & Management", "Comprehensive screening by a non-physician clinician of an individual, including vital signs, an age and gender appropriate history, coordination for ordering of laboratory/diagnostic procedures and questionnaire filling, new or established patient; 30-40", "Genito-urinary system and sex hormones", "Laboratory Services", "Fractures, Treatment Of", "Respiratory system", "Non-surgical cleansing of a wound without debridement, with or without local anesthesia, with or without the application of a surgical dressing: more than 48 sq inches / 300 sq centimeters.", "Per Diem- Bundled Base Payment of home care (Level 4- Complex ) *-A bundled payment that includes all medical services and transportation", "Room and Board: Suite", "Perdiem - Non- Medical Escort accommodation - Daily Rate", "Chemotherapy Drugs", "Prosthesis, Maxillofacial", "Per Diem- Bundled Base Payment of home care (Level 1- SIMPLE ) * -A bundled payment that includes all medical services and transportation -", "Examination And Diagnosis, Clinical Oral", "Temporary National Codes Established Medicaid", "Oral Surgery Procedures, Other", "CMS Hospital Outpatient Payment System", "Room and Board: Royal Suite", "Tele-consultation resulting in Prescription", "Orthodontic Services, Miscellaneous", "Perdiem - New-born Nursery - Daily Rate (Day 9 and more)", "Pathology & Laboratory", "Undefined services", "Perdiem - PICU - Daily Rate (Day 8 to 14)", "Diseases & Disorders Of The Eye", "Perdiem - SCBU - Daily Rate (Day 9 and more)", "Consultation - Consultant", "Antineoplastic and immunomodulating agents", "Perdiem - Haemodialysis (HD)", "Short Stay Room - Daily Rate", "Perdiem - Treatment or Observation Room - NOT inclusive of Laboratory and Radiology", "Surgical Incisions", "Gradient Compression Stockings", "Perdiem - Long Term Stay - Daily Rate - Simple Cases", "Crowns, Single Units\n(Only)", "Primary Dentition", "Short Stay Room - Hourly Rate", "Perdiem - Day Stay (Day care) Medical Case - Inclusive.", "Childbirth", "Perdiem - ICU/CCU - Daily Rate (Day 1 to 7)", "Antiparasitic products, insecticides and repellents", "Restorative Procedures, Overdentures", "Alcohol/Drug Abuse & Dependence", "Perdiem - NICU - Daily Rate (Day 22 and more)", "Systemic hormonal preparations, excluding sex hormones and insulins", "Restorations, Tooth Coloured/Plastic With/Without Silver Filings", "Orthotics", "Removals, (Extractions), Surgical", "Diseases & Disorders Of The Female Reproductive System", "Consultation - GP", "Room and Board: Shared Room", "Anatomic Modifications (Reshaping, Recontouring, Or Occlusal Modifications Of A Natural Tooth Or Teeth, Single Or Multiple Restorations, Or The Inter-Articulation Of The Teeth)", "Diseases & Disorders Of The Endocrine, Nutritional & Metabolic Systems", "Appliances, Retention, Orthodontic Retaining Appliances", "Transport Services Including Ambulance", "Master Cast Techniques", "Hemic & Lymphatic Systems", "Radiologic Guidance", "Perdiem - SCU - Daily Rate (Day 4 to 8)", "Medical and Surgical Supplies", "Diseases & Disorders Of The Nervous System", "Perdiem - PICU - Daily Rate (Day 1 to 7)", "Radiology", "Consultation - Specialist - Follow Up within one week", "Permanent Dentition", "Injuries, Poisonings & Toxic Effects Of Drugs", "Perdiem - Ward or Shared Room - Daily Rate (Day 1 to 3)", "Test/Analysis/Laboratory Procedures/Interpretation And/Or Reports", "Recovery Room - Hourly Rate", "Dentures, Adjustments\n(After Three Months Insertion Or By Other Than The Dentist Providing Prosthesis.)", "Operating Microscope", "Room Rate difference - Daily Rate (Day 1 and more ) - First Class Room", "Diseases & Disorders Of The Skin, Subcutaneous Tissue & Breast", "Tele-consultation resulting in Self Care", "Diagnostic Ultrasound", "Diseases & Disorders Of Blood, Blood Forming Organs, Immunological System", "Hearing Services", "Perdiem - Companion Accommodation", "Dermatologicals", "Perdiem - Automated Peritoneal Dialysis (APD).", "Bone/Joint Studies", "Perdiem - Long Term Stay - Daily Rate - Severe Cases", "Special Care Unit (SCU) or Adult Special-Care Unit (ASCU)", "Radiographs, Specialty Use Only", "Template, Surgical (Includes Diagnostic Wax-Up. Also Used To Locate And Orient Osseo-Integrated Implants)", "Mental Diseases & Disorders", "0", "Tele-consultation resulting in Emergency Management", "Restorative Services, Other", "Temporary National Codes Established by Private Payers", "Operating Room Services - General Classification", "Diseases & Disorders Of The Circulatory System", "Diseases & Disorders Of The Musculoskeletal System & Connective Tissue", "Diseases & Disorders Of The Digestive System", "Dentures, Repairs/Additions", "Periapical Services", "Helmets", "Mesostructures\n(A Separate Component Positioned Between The Head Of An Implant And The Final Restoration, Retained By Either A Cemented Post Or Screw)", "Perdiem - Long Term Stay - Daily Rate - Intermediate Cases", "Blood and blood forming organs", "Room and Board: Isolation Room", "Room and Board: Private Room Standard Suite", "Nursery - General Classification", "Male Genital System", "Temporomandibular Joint Dysfunctions, Treatment Of", "Room and Board: Ward", "Perdiem - Medical Escort accommodation - Daily Rate", "Professional Visits", "Unclassified Treatments", "Cardiovascular system", "Behavioural Health and/or Substance Abuse Treatment Services", "Alimentary tract and metabolism", "Comprehensive screening evaluation and management by clinician of an individual, including an age and gender appropriate history, questionnaire filling, examination, and ordering of laboratory/diagnostic procedures, new or established patient; 30-40 minut", "Consultation - GP - Follow Up within one week", "Nervous system", "Digestive System", "Delivery Room", "Space Maintainers\n(Includes The Design, Separation, Fabrication, Insertion And Where Applicable Initial Cementation And Removal)", "Perdiem - Day Stay (Day Care) Room - NOT inclusive of Laboratory and Radiology", "Enteral and Parenteral Therapy", "Consultation - Consultant - Follow Up within one week", "Fixed Bridge Retainers", "Perdiem - PICU - Daily Rate (Day 15 to 21)", "Perdiem - NICU - Daily Rate (Day 1 to 7)", "Administrative, Miscellaneous and Investigational", "Restorations, Pre-Fabricated, Full Coverage", "Diseases & Disorders Of The Male Reproductive System", "Surgical Excisions (Not In Conjunction With Tooth Removal, Including Biopsy)", "Dentures, Complete (Includes: Impressions, Initial And Final Jaw Relation Records, Try-In Evaluation And Check Records, Insertion And Adjustments, Including Three Months Post Insertion Care.)"], "Activity_Type": ["Service", "Drug", "Dental", "HCPCS", "DRG", "CPT"], "Unit_Type": ["AB -", "A +", "A -", "O -", "B -", "AB +", "O +", "B +"], "Blood_Bank_Center": ["SKMC Blood Bank", "Tawam Blood Bank"], "Disease": ["Other", "Other STIs", "Acute Encephalitis", "Viral Hepatitis B", "Giardia Lambia", "Syphilis", "Viral Meningitis", "Other Viral Hepatitis", "Scarlet Fever", "Corona Virus", "Extra Pulmonary Tuberculosis", "Brucellosis", "Other Food Poisoning", "Measles", "Tetanus", "Chickenpox", "Rubella", "Other STD", "Mumps", "Typhoid Fever", "Viral Hepatitis A", "Viral Hepatitis C", "Gonorrhoea", "Whooping Cough", "Scabies", "Acute Flaccid Paralysis", "Seasonal Influenza", "Pulmonary Tuberculosis", "Malaria"], "Cancer_site": ["Spinal Cord, Cranial Nerves And Other Parts Of Central Nervous System", "Lymphoid Leukaemia", "Heart, Mediastinum And Pleura", "Mesothelioma", "Breast", "Parotid Gland", "Endocrine Gland", "Bone And Articular Cartilage Of Other And Unspecified Sites", "Liver and intrahepatic bile ducts", "Secondary Respiratory And Digestive Organs", "Lymphoma", "Hypopharynx", "Follicular [Nodular] Non-Hodgkin'S Lymphoma", "Nasopharynx", "Other And Ill-Defined Sites In The Lip, Oral Cavity And Pharynx", "Other Leukaemias Of Specified Cell Type", "Skin", "Multiple myeloma", "Other And Unspecified Urinary Organs", "Gallbladder and extrahepatic ducts", "Stomach", "Trachea, bronchus and lung", "Malignant Neoplasms Of Independent (Primary) Multiple Sites", "Penis", "Corpus uteri", "Peripheral Nerves And Autonomic Nervous System", "Malignant Neoplasm Without Specification Of Site", "Myeloid Leukaemia", "Leukaemia Of Unspecified Cell Type", "Bladder", "Ureter", "Urinary Organs", "Melanoma of skin", "Prostate", "Oesophagus", "Other Cancer", "Bone", "Rectum", "Retroperitoneum And Peritoneum", "Diffuse", "Monocytic Leukaemia", "Testis", "Lip, oral cavity", "Piriform Sinus", "Kidney, renal pelvis and ureter", "Other And Ill-Defined Sites", "Base Of Tongue", "Cervix Uteri", "Other Malignant Neoplasms Of Skin", "Oropharynx", "Other And Unspecified Types Of Non-Hodgkin'S Lymphoma", "Other pharynx", "Kaposi sarcoma", "Other And Ill-Defined Sites In The Respiratory System And Intrathoracic Organs", "Multiple Myeloma And Malignant Plasma Cell Neoplasms", "Meninges", "Salivary Glands", "Corpus Uteri", "Rectosigmoid Junction", " Non-Hodgkin'S Lymphoma", "Brain, nervous system", "Eye And Adnexa", "Diffuse Non-Hodgkin'S Lymphoma", "Secondary Other Sites", "Bone And Articular Cartilage Of Limbs", "Tongue", "Uterus, Part Unspecified", "Other And Unspecified Parts Of Mouth", "Malignant Melanoma Of Skin", "Other And Ill-Defined Digestive Organs", "Other Connective And Soft Tissue", "Colon", "Other And Unspecified Parts Of Biliary Tract", "Brain", "Mouth", "Connective And Soft Tissue", "Non-melanoma skin cancer", "Leukaemia", "Cervix uteri", "Other And Unspecified Malignant Neoplasms Of Lymphoid, Haematopoietic And Related Tissue", "Liver And Intrahepatic Bile Ducts", "Colorectum", "Thyroid Gland", "Follicular-Non-Hodgkin'S Lymphoma", "Gallbladder", " Independent (Primary) Multiple Sites", "Anus And Anal Canal", "Biliary Tract", "Thyroid", "Other And Unspecified Major Salivary Glands", "Floor Of Mouth", "Adrenal Gland", "Pancreas", "Renal Pelvis", "Small Intestine", "Accessory Sinuses", "Bronchus And Lung", "Secondary", "Larynx", "Without Specification Of Site", "Ovary", "Kidney, Except Renal Pelvis", "Hodgkin'S Disease"]} \ No newline at end of file +{"Activity_Type_Chapter": ["Coronary Care Unit (CCU)", "Non-surgical cleansing of a wound without debridement, with or without local anesthesia, with or without the application of a surgical dressing: 16 sq inches / 100 sq centimeters or less.", "Perdiem - NICU - Daily Rate (Day 1 to 7)", "Perdiem - SCBU - Daily Rate (Day 4 to 8)", "Polishing", "Nursery - General Classification", "Room Rate difference - Daily Rate (Day 1 and more ) - Suite", "Operating Microscope", "Hemic & Lymphatic Systems", "Mesostructures\n(A Separate Component Positioned Between The Head Of An Implant And The Final Restoration, Retained By Either A Cemented Post Or Screw)", "Test/Analysis/Laboratory Procedures/Interpretation And/Or Reports", "Chemotherapy Drugs", "Root Canal Therapy\nTo Include: Treatment Plan, Clinical Procedures (I.E. Pulpectomy, Biomechanical Preparation, Chemotherapeutic Treatment And Obturation), With Appropriate Radiographs And Excluding Final Restoration.", "Periapical Services", "Medical and Surgical Supplies", "CMS Hospital Outpatient Payment System", "Room and Board: Shared Room", "Appliances, Retention, Orthodontic Retaining Appliances", "Room and Board: Ward", "Perdiem - Day Stay (Day Care) Room - NOT inclusive of Laboratory and Radiology", "Room and Board: Suite", "Room Rate difference - Daily Rate (Day 1 and more ) - VIP Room", "Tele-consultation resulting in Referral", "Consultation - GP", "Perdiem - NICU - Daily Rate (Day 15 to 21)", "Integumentary", "Gradient Compression Stockings", "Pathology & Laboratory", "Other Medical Services", "Vision Services", "Perdiem - NICU - Daily Rate (Day 22 and more)", "Perdiem - Medical Escort accommodation - Daily Rate", "Diagnostic Ultrasound", "Anatomic Modifications (Reshaping, Recontouring, Or Occlusal Modifications Of A Natural Tooth Or Teeth, Single Or Multiple Restorations, Or The Inter-Articulation Of The Teeth)", "Durable Medical Equipment", "Consultation - Consultant", "Infectious & Parasitic Diseases Of Systemic Or Unspecified Sites", "Perdiem - New-born Nursery - Daily Rate (Day 4 to 8)", "Systemic hormonal preparations, excluding sex hormones and insulins", "Primary Dentition", "Short Stay Room - Hourly Rate", "Template, Surgical (Includes Diagnostic Wax-Up. Also Used To Locate And Orient Osseo-Integrated Implants)", "Perdiem - PICU - Daily Rate (Day 8 to 14)", "Periodontal Services, Non Surgical", "Perdiem - Short Stay", "Per Diem- Bundled Base Payment of home care (Level 4- Complex ) *-A bundled payment that includes all medical services and transportation", "Mixed Dentition", "Restorations, Foil, Gold", "Diseases & Disorders Of The Skin, Subcutaneous Tissue & Breast", "Room and Board: Private Room Standard Suite", "Nervous System", "Transport Services Including Ambulance", "Diseases & Disorders Of The Hepatobiliary System & Pancreas", "Urinary System", "Operating Room Services - Every Additional 1/2 hr.", "Orthodontic Services, Miscellaneous", "Fixed Bridge Retainers", "Orthotics", "Diseases & Disorders Of The Digestive System", "Perdiem - ICU/CCU - Daily Rate (Day 8 to 14)", "Radiographs, Specialty Use Only", "Newborns & Other Neonates", "Temporary Procedures/Professional Services", "Alimentary tract and metabolism", "Diseases & Disorders Of The Circulatory System", "Anaesthesia", "Perdiem - Long Term Stay - Daily Rate - Severe Cases", "Perdiem - Day Stay (Day care) Medical Case - Inclusive.", "Perdiem - Haemodialysis (HD)", "Cardiovascular system", "Emergency Room - Hourly Rate", "Tele-consultation resulting in Self Care", "Mediastinum & Diaphragm", "Crowns, Single Units\n(Only)", "Non-surgical cleansing of a wound without debridement, with or without local anesthesia, with or without the application of a surgical dressing: between 16 sq inches / 100 sq centimeters and 48 sq inches / 300 sq centimeters .", "Permanent Dentition", "Musculoskeletal", "Perdiem - SCBU - Daily Rate (Day 9 and more)", "Professional Visits", "Diseases & Disorders Of The Musculoskeletal System & Connective Tissue", "Perdiem - Ward or Shared Room - Daily Rate (Day 1 to 3)", "Alcohol/Drug Abuse & Dependence", "Perdiem - New-born Nursery - Daily Rate (Day 1 to 3)", "Per Diem- Bundled Base Payment of home care (Level 1- SIMPLE ) * -A bundled payment that includes all medical services and transportation -", "Perdiem - SCU - Daily Rate (Day 4 to 8)", "Recontouring Of Retainer/Pontics,\n(Of Existing Bridgework)", "Prosthesis, Maxillofacial", "Respiratory System", "Antiinfectives for systemic use", "Oral Surgery Procedures, Other", "Temporary Codes assigned to DME Regional Carriers", "Anesthesia", "Perdiem - Treatment or Observation Room - NOT inclusive of Laboratory and Radiology", "Undefined services", "Nuclear Medicine", "Perdiem - Continuous Ambulatory Peritoneal Dialysis (CAPD).", "Educational services rendered by Clinician (Physician or Non-Physician) to patient opting out of the Comprehensive Screening, new or established patient; 10-15 minutes.", "Laboratory And Expense Procedures (This Code Is Used In Conjunction With The \"+ L\" And \"+ E\" Designation Following Specific Codes In The Guide. The Addition Of These Codes Are To Facilitate Computer Or Manual Input For Third Party Claims Processing, Perso", "Space Maintainers\n(Includes The Design, Separation, Fabrication, Insertion And Where Applicable Initial Cementation And Removal)", "Operating Room Services - First Hour", "Diseases & Disorders Of The Urinary Tract", "Hearing Services", "Tele-consultation resulting in Prescription", "Unclassified Treatments", "Endocrine System", "Perdiem - SCBU - Daily Rate (Day 1 to 3)", "Paediatric intensive care Unit (PICU)", "DRG Add-on Payment for High Cost Consumables", "Restorations, Amalgam", "Special Care Unit (SCU) or Adult Special-Care Unit (ASCU)", "Pontics, Bridge", "Surgical Excisions (Not In Conjunction With Tooth Removal, Including Biopsy)", "Non-surgical cleansing of a wound without debridement, with or without local anesthesia, with or without the application of a surgical dressing: more than 48 sq inches / 300 sq centimeters.", "Evaluation & Management", "Case Presentation/ Treatment Planning", "Genito-urinary system and sex hormones", "Antineoplastic and immunomodulating agents", "Consultation - Specialist", "Perdiem - Long Term Stay - Daily Rate - Simple Cases", "Musculo-skeletal system", "Dentures, Partial, Cast With Acrylic Base", "Dermatologicals", "Room and Board: Isolation Room", "Remodelling And Recontouring Oral Tissues In Preparation For Removable Prostheses (To Include Codes 73110, 73120, 73140, 73150, 73160, 73170, 73180)", "Bone/Joint Studies", "Dentures, Replication, Relining And Rebasing", "Diseases & Disorders Of The Endocrine, Nutritional & Metabolic Systems", "Medical Ambulatory Visits", "Diseases & Disorders Of The Ear, Nose, Mouth & Throat", "Appliances, Active, For Tooth Guidance Or Minor Tooth Movement", "Perdiem - ICU/CCU - Daily Rate (Day 22 and more)", "Restorations, Tooth Coloured/Plastic With/Without Silver Filings", "Endodontic, Procedures, Miscellaneous", "Fractures, Treatment Of", "Perdiem - Long Term Stay - Daily Rate - Intensive Cases", "Short Stay Room - Daily Rate", "Perdiem - PICU - Daily Rate (Day 1 to 7)", "Male Genital System", "Surgical Incisions", "Temporary National Codes Established by Private Payers", "Perdiem - Day Stay (Day care) - Inclusive.", "Perdiem - New-born Nursery - Daily Rate (Day 9 and more)", "Maternity Care & Delivery", "Auditory System", "Room and Board: First Class Room", "Perdiem - ICU/CCU - Daily Rate (Day 1 to 7)", "Fixed Prosthetics, Other Services", "Dentures, Partial (Fabricated In Conjunction With A Denturist)", "Catheterization Lab", "Delivery Room", "Outlier Payment", "Tele-consultation resulting in Emergency Management", "Eye & Ocular Adnexa", "Room Rate difference - Daily Rate (Day 1 and more ) - Royal Suite", "Intensive Care Unit (ICU)", "Per Diem- Bundled Base Payment of home care (Level 3- Intensive) * -A bundled payment that includes all medical services and tra", "Restorative Services, Other", "Perdiem - SCU - Daily Rate (Day 9 and more)", "Recovery Room - Hourly Rate", "Appliances", "Diseases & Disorders Of The Male Reproductive System", "Removals, (Extractions), Erupted Teeth", "Respiratory Durable Medical Equipment, Inexpensive and Routinely Purchased", "Comprehensive screening evaluation and management by clinician of an individual, including an age and gender appropriate history, questionnaire filling, examination, and ordering of laboratory/diagnostic procedures, new or established patient; 30-40 minut", "Special Care Baby Unit (SCBU)", "Day Stay (Day care) Room - Daily Rate", "Bleaching Vital", "Periodontal Services, Surgical (Includes Local Anaesthetic, Suturing And The Placement And Removal Of Initial Surgical Dressing. A Surgical Site Is An Area That Lends Itself To One Or More Procedures. It Is Considered To Include A Full Quadrant, Sextant O", "Perdiem - NICU - Daily Rate (Day 8 to 14)", "Diseases & Disorders Of The Female Reproductive System", "Antiparasitic products, insecticides and repellents", "Perdiem - Long Term Stay - Daily Rate - Intermediate Cases", "Consultation - GP - Follow Up within one week", "Cardiovascular System", "Childbirth", "Factors Influencing Health Status & Other Contacts With Health Services", "Respiratory system", "Perdiem - Automated Peritoneal Dialysis (APD).", "Pulp Chamber, Treatment Of, (Excluding Final Restoration)", "Digestive System", "Restorations, Inlays, Onlays, Pins And Posts", "Diseases & Disorders Of Blood, Blood Forming Organs, Immunological System", "Temporary Codes Assigned by CMS", "Room and Board: VIP Room", "Perdiem - Non- Medical Escort accommodation - Daily Rate", "Injuries, Poisonings & Toxic Effects Of Drugs", "Radiologic Guidance", "Behavioural Health and/or Substance Abuse Treatment Services", "Drugs Other Than Chemotherapy", "Various", "Diseases & Disorders Of The Myeloproliferative System & Poorly Differentiated Neoplasms", "Dentures, Complete (Includes: Impressions, Initial And Final Jaw Relation Records, Try-In Evaluation And Check Records, Insertion And Adjustments, Including Three Months Post Insertion Care.)", "Perdiem - Ward or Shared Room - Daily Rate (Day 8 and more)", "Operating Room Services - General Classification", "Perdiem - PICU - Daily Rate (Day 22 and more)", "Radiographs (Including Radiographic Examination And Diagnosis And Interpretation)", "Restorative Procedures, Overdentures", "Consultation - Specialist - Follow Up within one week", "Helmets", "Perdiem - SCU - Daily Rate (Day 1 to 3)", "Perdiem - Companion Accommodation", "0", "Room Rate difference - Daily Rate (Day 1 and more ) - Isolation Room", "Prosthondics, Removable, Fabricated In Conjunction With A Denturist", "Restorations, Pre-Fabricated, Full Coverage", "Dental Procedures", "Temporomandibular Joint Dysfunctions, Treatment Of", "Perdiem - Ward or Shared Room - Daily Rate (Day 4 to 8)", "Diseases & Disorders Of The Eye", "Administrative, Miscellaneous and Investigational", "Radiation Oncology", "Laboratory Services", "Neonatal Intensive Care Unit (NICU)", "Consultation - Consultant - Follow Up within one week", "Per Diem- Bundled Base Payment of home care (Level 2- Intermediate ) * -A bundled payment that includes all medical services and transportation - Su", "Periodontal Procedures, Adjunctive (When Per Joint Is Designated, The Corresponding Tooth Code Is Represented By The Mesial Of The Tooth Involved, Except At The Midline, Where The Tooth To The Right Of The Joint Is Utilized.)", "Female Genital System", "Perdiem - ICU/CCU - Daily Rate (Day 15 to 21)", "Professional Services", "Nervous system", "Prosthetics", "Restoration", "Diseases & Disorders Of The Respiratory System", "Operating Room Services - Minor Surgery", "Perdiem - PICU - Daily Rate (Day 15 to 21)", "Perdiem - Day Stay (Day care) Surgical Case - Inclusive.", "Frequent Preventative Services Billed On A Time Unit Basis. Services Covered Under This Code Are: Polishing, Scaling, Root Planing, Nutritional Counselling, Oral Hygiene Instruction/Plaque Control And Application Of An Antimicrobial Agent\n***Provinces Usi", "Dentures, Partial, Acrylic", "Repairs", "Room Rate difference - Daily Rate (Day 1 and more ) - First Class Room", "Room and Board: Royal Suite", "Diseases & Disorders Of The Nervous System", "Drugs/Medication, Dispensing", "Removals, (Extractions), Surgical", "Temporary National Codes Established Medicaid", "Radiology", "Enteral and Parenteral Therapy", "Master Cast Techniques", "Examination And Diagnosis, Clinical Oral", "Comprehensive screening by a non-physician clinician of an individual, including vital signs, an age and gender appropriate history, coordination for ordering of laboratory/diagnostic procedures and questionnaire filling, new or established patient; 30-40", "Dentures, Repairs/Additions", "Maxillofacial Deformities, Treatment Of", "Blood and blood forming organs", "General", "Mental Diseases & Disorders", "Periodontal Services, Miscellaneous", "Sensory organs", "Breast Mammography", "Preventive Services, Other", "Dentures, Adjustments\n(After Three Months Insertion Or By Other Than The Dentist Providing Prosthesis.)"], "Activity_Type": ["HCPCS", "Service", "Dental", "Drug", "CPT", "DRG"], "Unit_Type": ["B -", "B +", "AB -", "O -", "O +", "A +", "A -", "AB +"], "Blood_Bank_Center": ["Tawam Blood Bank", "SKMC Blood Bank"], "Disease": ["Scarlet Fever", "Viral Hepatitis C", "Tetanus", "Seasonal Influenza", "Giardia Lambia", "Acute Flaccid Paralysis", "Typhoid Fever", "Viral Hepatitis A", "Malaria", "Other STIs", "Scabies", "Mumps", "Brucellosis", "Pulmonary Tuberculosis", "Other Viral Hepatitis", "Corona Virus", "Rubella", "Acute Encephalitis", "Other", "Viral Meningitis", "Chickenpox", "Viral Hepatitis B", "Measles", "Other Food Poisoning", "Other STD", "Whooping Cough", "Gonorrhoea", "Syphilis", "Extra Pulmonary Tuberculosis"], "Cancer_site": ["Malignant Melanoma Of Skin", "Peripheral Nerves And Autonomic Nervous System", "Gallbladder and extrahepatic ducts", "Tongue", "Multiple Myeloma And Malignant Plasma Cell Neoplasms", "Small Intestine", "Salivary Glands", "Other And Unspecified Types Of Non-Hodgkin'S Lymphoma", "Ovary", "Prostate", "Renal Pelvis", "Multiple myeloma", "Other And Unspecified Major Salivary Glands", "Corpus uteri", "Kidney, renal pelvis and ureter", "Corpus Uteri", "Trachea, bronchus and lung", "Myeloid Leukaemia", "Lymphoid Leukaemia", "Other Connective And Soft Tissue", "Follicular [Nodular] Non-Hodgkin'S Lymphoma", "Diffuse", "Anus And Anal Canal", "Secondary Other Sites", "Kidney, Except Renal Pelvis", "Lymphoma", "Penis", "Skin", "Other And Ill-Defined Sites", "Bronchus And Lung", "Mouth", "Non-melanoma skin cancer", "Cervix uteri", "Base Of Tongue", "Pancreas", "Bladder", "Uterus, Part Unspecified", "Other And Ill-Defined Digestive Organs", "Hodgkin'S Disease", "Lip, oral cavity", "Bone And Articular Cartilage Of Limbs", "Bone", "Liver and intrahepatic bile ducts", "Brain, nervous system", "Larynx", "Meninges", "Piriform Sinus", "Spinal Cord, Cranial Nerves And Other Parts Of Central Nervous System", "Secondary Respiratory And Digestive Organs", "Other And Unspecified Urinary Organs", "Colon", "Bone And Articular Cartilage Of Other And Unspecified Sites", "Oesophagus", "Melanoma of skin", "Cervix Uteri", "Gallbladder", "Colorectum", "Nasopharynx", "Other And Unspecified Malignant Neoplasms Of Lymphoid, Haematopoietic And Related Tissue", "Testis", "Accessory Sinuses", "Leukaemia", "Secondary", " Non-Hodgkin'S Lymphoma", "Parotid Gland", "Endocrine Gland", "Adrenal Gland", "Other Cancer", "Kaposi sarcoma", "Other Leukaemias Of Specified Cell Type", "Other Malignant Neoplasms Of Skin", "Ureter", "Heart, Mediastinum And Pleura", "Breast", "Liver And Intrahepatic Bile Ducts", "Diffuse Non-Hodgkin'S Lymphoma", "Thyroid", "Other pharynx", "Other And Ill-Defined Sites In The Respiratory System And Intrathoracic Organs", "Hypopharynx", " Independent (Primary) Multiple Sites", "Eye And Adnexa", "Malignant Neoplasm Without Specification Of Site", "Connective And Soft Tissue", "Stomach", "Monocytic Leukaemia", "Other And Unspecified Parts Of Biliary Tract", "Other And Unspecified Parts Of Mouth", "Thyroid Gland", "Rectosigmoid Junction", "Mesothelioma", "Leukaemia Of Unspecified Cell Type", "Retroperitoneum And Peritoneum", "Other And Ill-Defined Sites In The Lip, Oral Cavity And Pharynx", "Biliary Tract", "Malignant Neoplasms Of Independent (Primary) Multiple Sites", "Rectum", "Oropharynx", "Without Specification Of Site", "Urinary Organs", "Follicular-Non-Hodgkin'S Lymphoma", "Brain", "Floor Of Mouth"]} \ No newline at end of file diff --git a/tableqa/vocab.json b/tableqa/vocab.json index ee48632..d804a9e 100644 --- a/tableqa/vocab.json +++ b/tableqa/vocab.json @@ -1 +1 @@ -{"Activities - Data.csv": ["type", "year", "activities", "chapter", "index", "percentage", "activity", "percent", "per", "value"], "Blood Banks - Data.csv": ["type", "count", "bank", "year", "units", "banks", "group", "center", "blood", "unit"], "GDP at current prices (AED)-Production Approach.csv": ["price", "year", "domestic", "product", "current", "gdp", "gross", "value"], "Communicable Diseases - Data.csv": ["disease", "nationality", "year", "age", "sickness", "cases", "index", "communicable", "illness", "diseases", "occurances", "number", "total"], "Cancer Death - Data.csv": ["type", "count", "dead", "nationality", "year", "age", "gender", "cancer", "end", "index", "death", "died", "site", "location"]} \ No newline at end of file +{"Activities - Data.csv": ["value", "chapter", "index", "percent", "year", "activities", "type", "per", "percentage", "activity"], "Blood Banks - Data.csv": ["group", "banks", "units", "count", "center", "bank", "year", "unit", "type", "blood"], "GDP at current prices (AED)-Production Approach.csv": ["value", "product", "year", "current", "gdp", "domestic", "price", "gross"], "Communicable Diseases - Data.csv": ["disease", "age", "sickness", "index", "occurances", "total", "diseases", "number", "year", "nationality", "illness", "communicable", "cases"], "Cancer Death - Data.csv": ["cancer", "age", "count", "died", "end", "index", "gender", "site", "year", "type", "location", "nationality", "death", "dead"]} \ No newline at end of file From 08620ff8822d6c8227fc0a45889c3e35a3151671 Mon Sep 17 00:00:00 2001 From: abhijithneilabraham Date: Sat, 29 Aug 2020 07:54:15 +0530 Subject: [PATCH 2/2] bug fixes --- tableqa/clauses.py | 23 ++++++++++------------- tableqa/nlp.py | 15 +++++++++++++-- tableqa/values.json | 2 +- tableqa/vocab.json | 2 +- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/tableqa/clauses.py b/tableqa/clauses.py index d59f7fc..aa61850 100755 --- a/tableqa/clauses.py +++ b/tableqa/clauses.py @@ -1,18 +1,15 @@ -from nlp import qa -import tensorflow as tf -class Clause: - def __init__(self): +#from nlp import qa +# class Clause: +# def __init__(self): - self.base_q="what is {} here" - self.types={"the entity":'SELECT {} FROM {}', "the maximum":'SELECT MAX({}) FROM {}', "the minimum":'SELECT MIN({}) FROM {}', "counted":'SELECT COUNT({}) FROM {}', "summed":'SELECT SUM({}) FROM {}', "averaged":'SELECT AVG({}) FROM {}'} +# self.base_q="what is {} here" +# self.types={"the entity":'SELECT {} FROM {}', "the maximum":'SELECT MAX({}) FROM {}', "the minimum":'SELECT MIN({}) FROM {}', "counted":'SELECT COUNT({}) FROM {}', "summed":'SELECT SUM({}) FROM {}', "averaged":'SELECT AVG({}) FROM {}'} - def adapt(self,q,inttype=False,priority=False): - scores={} - validated_scores={} - total_scores={} - for k,v in self.types.items(): - scores[k]=tf.convert_to_tensor(qa(q,self.base_q.format(k),return_score=True)[1]) - return self.types[max(scores, key=scores.get)] +# def adapt(self,q,inttype=False,priority=False): +# scores={} +# for k,v in self.types.items(): +# scores[k]=qa(q,self.base_q.format(k),return_score=True)[1] +# return self.types[max(scores, key=scores.get)] \ No newline at end of file diff --git a/tableqa/nlp.py b/tableqa/nlp.py index 7ed2d76..f5c2787 100644 --- a/tableqa/nlp.py +++ b/tableqa/nlp.py @@ -2,12 +2,10 @@ from data_utils import data_utils import os from transformers import TFBertForQuestionAnswering, BertTokenizer -from transformers import TFAutoModelForSequenceClassification, AutoTokenizer import tensorflow as tf from rake_nltk import Rake import column_types import json -from clauses import Clause from conditionmaps import conditions @@ -165,6 +163,19 @@ def _window_overlap(s1, e1, s2, e2): return s2 <= e1 if s1