diff --git a/data/adoption/adoption_dfj.ipynb b/data/adoption/adoption_dfj.ipynb index f11f99f..301f36c 100644 --- a/data/adoption/adoption_dfj.ipynb +++ b/data/adoption/adoption_dfj.ipynb @@ -84,10 +84,10 @@ "outputs": [], "source": [ "#this is the database we will be extracting from\n", - "database = \"familyman_live_v4\" \n", + "database = \"stg_familyman\" \n", "\n", "#this extracts the latest snapshot from athena\n", - "get_snapshot_date = f\"SELECT mojap_snapshot_date from {database}.events order by mojap_snapshot_date desc limit 1\"\n", + "get_snapshot_date = f\"SELECT mojap_snapshot_date from {database}.stg_events order by mojap_snapshot_date desc limit 1\"\n", "snapshot_date = str(pydb.read_sql_query(get_snapshot_date)['mojap_snapshot_date'].values[0])\n", "\n", "#this extracts the November snapshot from athena\n", diff --git a/data/adoption/adoption_extraction.ipynb b/data/adoption/adoption_extraction.ipynb index 3fc40b7..0d4e279 100644 --- a/data/adoption/adoption_extraction.ipynb +++ b/data/adoption/adoption_extraction.ipynb @@ -124,10 +124,10 @@ "outputs": [], "source": [ "#this is the database we will be extracting from\n", - "database = \"familyman_live_v4\" \n", + "database = \"stg_familyman\" \n", "\n", "#this extracts the latest snapshot from athena\n", - "get_snapshot_date = f\"SELECT mojap_snapshot_date from {database}.events order by mojap_snapshot_date desc limit 1\"\n", + "get_snapshot_date = f\"SELECT mojap_snapshot_date from {database}.stg_events order by mojap_snapshot_date desc limit 1\"\n", "snapshot_date = str(pydb.read_sql_query(get_snapshot_date)['mojap_snapshot_date'].values[0])\n", "\n", "#this extracts the November snapshot from athena\n", @@ -215,10 +215,10 @@ " P.DOB,\n", " P.GENDER,\n", " P.MOJAP_SNAPSHOT_DATE\n", - "FROM {database}.parties P\n", - "INNER JOIN {database}.roles R\n", + "FROM {database}.stg_parties P\n", + "INNER JOIN {database}.stg_roles R\n", "ON P.PARTY = R.PARTY\n", - "INNER JOIN {database}.role_fields F\n", + "INNER JOIN {database}.stg_role_fields F\n", "ON R.ROLE = F.ROLE\n", "WHERE ((R.ROLE_MODEL = 'APLZ' --Applicant role model code\n", "AND F.FIELD_MODEL = 'APLZ_RTC') -- Applicant relationship to child field model\n", @@ -312,12 +312,12 @@ " F.VALUE AS App_Type,\n", " date_parse(F1.VALUE, '%Y-%m-%d') AS Date_of_issue,\n", " F2.VALUE AS High_court\n", - "FROM {database}.events E\n", - "INNER JOIN {database}.event_fields F2\n", + "FROM {database}.stg_events E\n", + "INNER JOIN {database}.stg_event_fields F2\n", "ON E.EVENT = F2.EVENT\n", - "INNER JOIN {database}.event_fields F1\n", + "INNER JOIN {database}.stg_event_fields F1\n", "ON E.EVENT = F1.EVENT\n", - "INNER JOIN {database}.event_fields F\n", + "INNER JOIN {database}.stg_event_fields F\n", "ON E.EVENT = F.EVENT\n", "WHERE (E.EVENT_MODEL = 'U21' --adoption proceedings issued\n", "AND F.FIELD_MODEL = 'U21_1' -- application type\n", @@ -354,8 +354,8 @@ " END AS Date_of_issue,\n", " 'N' AS High_court\n", "FROM\n", - " {database}.events E\n", - " INNER JOIN {database}.event_fields F\n", + " {database}.stg_events E\n", + " INNER JOIN {database}.stg_event_fields F\n", " ON E.EVENT = F.EVENT\n", "WHERE\n", " F.FIELD_MODEL = 'G50_AT' -- type of application\n", @@ -459,10 +459,10 @@ " WHEN 'BM08Z09028' THEN '2008-09-19'\n", " ELSE F1.VALUE\n", " END AS Case_issue_date2\n", - "FROM {database}.case_fields F\n", - "INNER JOIN {database}.case_fields F2\n", + "FROM {database}.stg_case_fields F\n", + "INNER JOIN {database}.stg_case_fields F2\n", "ON F.CASE_NUMBER = F2.CASE_NUMBER\n", - "INNER JOIN {database}.case_fields F1\n", + "INNER JOIN {database}.stg_case_fields F1\n", "ON F.CASE_NUMBER = F1.CASE_NUMBER\n", "WHERE F.FIELD_MODEL = 'FM3A_AC' -- adoption contested\n", "AND F1.FIELD_MODEL = 'FM3A_DOI' -- date of issue\n", @@ -545,7 +545,7 @@ " ELSE E.RECEIPT_DATE \n", " END AS Receipt_date2,\n", " E.EVENT_MODEL\n", - "FROM {database}.events E\n", + "FROM {database}.stg_events E\n", "WHERE ((E.EVENT_MODEL IN ('A70', 'A71', 'A72', 'A73', 'A74', 'A75', 'A76', 'A77', 'A78', 'A79', 'A80', 'A81', 'A12', 'A13', 'A15') -- all adoption event model codes\n", "AND E.ERROR = 'N')\n", "OR (E.EVENT_MODEL IN ('G63', 'ORDREF', 'ORDNOM') -- application withdrawn, order refused, no order made\n", @@ -625,7 +625,7 @@ "SELECT EVENT,\n", "FIELD_MODEL,\n", "VALUE\n", - "FROM {database}.event_fields E\n", + "FROM {database}.stg_event_fields E\n", "WHERE (FIELD_MODEL LIKE 'A1%'\n", " OR FIELD_MODEL LIKE 'A7%'\n", " OR FIELD_MODEL IN ('G63_1', 'ORDNOM_5', 'ORDREF_5', 'A80_4', 'A81_5')) -- extracts all adoption related fields from the event_fields table\n", diff --git a/data/adoption/adoption_timeliness.ipynb b/data/adoption/adoption_timeliness.ipynb index 6dc174f..013cdaa 100644 --- a/data/adoption/adoption_timeliness.ipynb +++ b/data/adoption/adoption_timeliness.ipynb @@ -96,10 +96,10 @@ "outputs": [], "source": [ "#this is the database we will be extracting from\n", - "database = \"familyman_live_v4\" \n", + "database = \"stg_familyman\" \n", "\n", "#this extracts the latest snapshot from athena\n", - "get_snapshot_date = f\"SELECT mojap_snapshot_date from {database}.events order by mojap_snapshot_date desc limit 1\"\n", + "get_snapshot_date = f\"SELECT mojap_snapshot_date from {database}.stg_events order by mojap_snapshot_date desc limit 1\"\n", "snapshot_date = str(pydb.read_sql_query(get_snapshot_date)['mojap_snapshot_date'].values[0])\n", "\n", "#this extracts the November snapshot from athena\n", @@ -648,26 +648,26 @@ "CREATE TABLE IF NOT EXISTS fcsq.Adopt_Applicant_Info\n", "WITH (format = 'PARQUET', external_location = 's3://alpha-family-data/fcsq_processing/Adoption/Adopt_Applicant_Info') AS\n", " SELECT DISTINCT\n", - " {database}.roles.ROLE, \n", - " {database}.roles.REPRESENTATIVE_ROLE, \n", - " {database}.roles.ROLE_MODEL, \n", - " {database}.roles.PARTY, \n", - " {database}.roles.CASE_NUMBER, \n", - " {database}.parties.PERSON_GIVEN_FIRST_NAME, \n", - " {database}.parties.PERSON_FAMILY_NAME, \n", - " {database}.parties.COMPANY, \n", - " {database}.addresses.POSTCODE, \n", - " {database}.parties.GENDER, \n", - " {database}.roles.DELETE_FLAG\n", + " {database}.stg_roles.ROLE, \n", + " {database}.stg_roles.REPRESENTATIVE_ROLE, \n", + " {database}.stg_roles.ROLE_MODEL, \n", + " {database}.stg_roles.PARTY, \n", + " {database}.stg_roles.CASE_NUMBER, \n", + " {database}.stg_parties.PERSON_GIVEN_FIRST_NAME, \n", + " {database}.stg_parties.PERSON_FAMILY_NAME, \n", + " {database}.stg_parties.COMPANY, \n", + " {database}.stg_addresses.POSTCODE, \n", + " {database}.stg_parties.GENDER, \n", + " {database}.stg_roles.DELETE_FLAG\n", "FROM \n", - " ({database}.roles INNER JOIN {database}.parties ON {database}.roles.PARTY = {database}.parties.PARTY) \n", - " INNER JOIN {database}.addresses ON {database}.roles.ADDRESS = {database}.addresses.ADDRESS\n", + " ({database}.stg_roles INNER JOIN {database}.stg_parties ON {database}.stg_roles.PARTY = {database}.stg_parties.PARTY) \n", + " INNER JOIN {database}.stg_addresses ON {database}.stg_roles.ADDRESS = {database}.stg_addresses.ADDRESS\n", "WHERE \n", - " (((({database}.roles.ROLE_MODEL)= 'APLZ') AND (({database}.roles.DELETE_FLAG)= 'N')) \n", - " OR ((({database}.roles.ROLE_MODEL)= 'APLA') AND (({database}.roles.DELETE_FLAG)= 'N')))\n", - " AND {database}.roles.mojap_snapshot_date = date '{snapshot_date}'\n", - " AND {database}.parties.mojap_snapshot_date = date '{snapshot_date}'\n", - " AND {database}.addresses.mojap_snapshot_date = date '{snapshot_date}';\n", + " (((({database}.stg_roles.ROLE_MODEL)= 'APLZ') AND (({database}.stg_roles.DELETE_FLAG)= 'N')) \n", + " OR ((({database}.stg_roles.ROLE_MODEL)= 'APLA') AND (({database}.stg_roles.DELETE_FLAG)= 'N')))\n", + " AND {database}.stg_roles.mojap_snapshot_date = date '{snapshot_date}'\n", + " AND {database}.stg_parties.mojap_snapshot_date = date '{snapshot_date}'\n", + " AND {database}.stg_addresses.mojap_snapshot_date = date '{snapshot_date}';\n", "\"\"\"\n", "\n", "pydb.start_query_execution_and_wait(create_Adopt_Applicant_Info);\n", @@ -756,23 +756,23 @@ "CREATE TABLE IF NOT EXISTS fcsq.adopt_respondent_info\n", "WITH (format = 'PARQUET', external_location = 's3://alpha-family-data/fcsq_processing/Adoption/adopt_respondent_info') AS\n", "SELECT DISTINCT\n", - " {database}.roles.ROLE, \n", - " {database}.roles.REPRESENTATIVE_ROLE, \n", - " {database}.roles.ROLE_MODEL, \n", - " {database}.roles.PARTY, \n", - " {database}.roles.CASE_NUMBER, \n", - " {database}.parties.GENDER, \n", - " {database}.addresses.POSTCODE, \n", - " {database}.roles.DELETE_FLAG\n", + " {database}.stg_roles.ROLE, \n", + " {database}.stg_roles.REPRESENTATIVE_ROLE, \n", + " {database}.stg_roles.ROLE_MODEL, \n", + " {database}.stg_roles.PARTY, \n", + " {database}.stg_roles.CASE_NUMBER, \n", + " {database}.stg_parties.GENDER, \n", + " {database}.stg_addresses.POSTCODE, \n", + " {database}.stg_roles.DELETE_FLAG\n", "FROM \n", - " ({database}.roles INNER JOIN {database}.parties ON {database}.roles.PARTY = {database}.parties.PARTY) \n", - " LEFT JOIN {database}.addresses ON {database}.roles.ADDRESS = {database}.addresses.ADDRESS\n", + " ({database}.stg_roles INNER JOIN {database}.stg_parties ON {database}.stg_roles.PARTY = {database}.stg_parties.PARTY) \n", + " LEFT JOIN {database}.stg_addresses ON {database}.stg_roles.ADDRESS = {database}.stg_addresses.ADDRESS\n", "WHERE \n", - " (((({database}.roles.ROLE_MODEL)='RSPA') AND (({database}.roles.DELETE_FLAG)='N')) \n", - " OR ((({database}.roles.ROLE_MODEL)='RSPZ') AND (({database}.roles.DELETE_FLAG)='N')))\n", - " AND {database}.roles.mojap_snapshot_date = date '{snapshot_date}'\n", - " AND {database}.parties.mojap_snapshot_date = date '{snapshot_date}'\n", - " AND {database}.addresses.mojap_snapshot_date = date '{snapshot_date}';\n", + " (((({database}.stg_roles.ROLE_MODEL)='RSPA') AND (({database}.stg_roles.DELETE_FLAG)='N')) \n", + " OR ((({database}.stg_roles.ROLE_MODEL)='RSPZ') AND (({database}.stg_roles.DELETE_FLAG)='N')))\n", + " AND {database}.stg_roles.mojap_snapshot_date = date '{snapshot_date}'\n", + " AND {database}.stg_parties.mojap_snapshot_date = date '{snapshot_date}'\n", + " AND {database}.stg_addresses.mojap_snapshot_date = date '{snapshot_date}';\n", "\"\"\"\n", "\n", "pydb.start_query_execution_and_wait(create_adopt_respondent_info);\n", @@ -1081,22 +1081,22 @@ "create_Adopt_Hearing_Events = f\"\"\"\n", "CREATE TABLE IF NOT EXISTS fcsq.Adopt_Hearing_Events\n", "WITH (format = 'PARQUET', external_location = 's3://alpha-family-data/fcsq_processing/Adoption/Adopt_Hearing_Events') AS\n", - "SELECT {database}.hearings.EVENT,\n", - " {database}.hearings.VACATED_FLAG,\n", - " {database}.hearings.HEARING_TYPE,\n", - " {database}.hearings.HEARING_DATE,\n", - " {database}.events.RECEIPT_DATE,\n", - " {database}.events.ERROR,\n", - " {database}.events.CASE_NUMBER,\n", - " {database}.events.EVENT_MODEL\n", - "FROM {database}.hearings\n", - "INNER JOIN {database}.events\n", - "ON {database}.hearings.EVENT = {database}.events.EVENT\n", - "WHERE {database}.hearings.VACATED_FLAG IS NULL\n", - "AND {database}.events.ERROR = 'N'\n", + "SELECT {database}.stg_hearings.EVENT,\n", + " {database}.stg_hearings.VACATED_FLAG,\n", + " {database}.stg_hearings.HEARING_TYPE,\n", + " {database}.stg_hearings.HEARING_DATE,\n", + " {database}.stg_events.RECEIPT_DATE,\n", + " {database}.stg_events.ERROR,\n", + " {database}.stg_events.CASE_NUMBER,\n", + " {database}.stg_events.EVENT_MODEL\n", + "FROM {database}.stg_hearings\n", + "INNER JOIN {database}.stg_events\n", + "ON {database}.stg_hearings.EVENT = {database}.stg_events.EVENT\n", + "WHERE {database}.stg_hearings.VACATED_FLAG IS NULL\n", + "AND {database}.stg_events.ERROR = 'N'\n", "AND HEARING_DATE > date_parse('31-12-2009 00:00:00', '%d-%m-%Y %H:%i:%s')\n", "AND (substring(case_number,5,1)='A' OR substring(case_number,5,1)='Z')\n", - "AND {database}.hearings.mojap_snapshot_date = date '{snapshot_date}' and {database}.events.mojap_snapshot_date = date '{snapshot_date}';\n", + "AND {database}.stg_hearings.mojap_snapshot_date = date '{snapshot_date}' and {database}.stg_events.mojap_snapshot_date = date '{snapshot_date}';\n", "\"\"\"\n", "\n", "pydb.start_query_execution_and_wait(create_Adopt_Hearing_Events);\n",