From b2ae52ec0557526b5c82ecfb187af70217951b37 Mon Sep 17 00:00:00 2001 From: "Hassan D. M. Sambo" Date: Wed, 13 Dec 2023 13:58:28 -0500 Subject: [PATCH 1/4] 2842 improve data migration code base (#2975) * #2842 Refactored code, Cleaned code, Improved logic * #2842 Added tests to core logic * #2842 Regenerated workbook to unsure the logic works as intended * #2842 Remove logic for scanning and uploading workbooks to s3 from migration process + code improvement * #2842 More improvement --- .../additional-eins-workbook-177310.xlsx | Bin 225121 -> 225120 bytes .../additional-ueis-workbook-177310.xlsx | Bin 225118 -> 225117 bytes .../audit-findings-text-workbook-177310.xlsx | Bin 267321 -> 267319 bytes ...orrective-action-plan-workbook-177310.xlsx | Bin 264338 -> 264337 bytes ...awards-audit-findings-workbook-177310.xlsx | Bin 1125520 -> 1125518 bytes .../federal-awards-workbook-177310.xlsx | Bin 1550706 -> 1550705 bytes .../notes-to-sefa-workbook-177310.xlsx | Bin 264834 -> 264834 bytes .../secondary-auditors-workbook-177310.xlsx | Bin 295038 -> 295037 bytes .../additional-eins-workbook-180818.xlsx | Bin 225123 -> 225123 bytes .../additional-ueis-workbook-180818.xlsx | Bin 225119 -> 225118 bytes .../audit-findings-text-workbook-180818.xlsx | Bin 263675 -> 263674 bytes ...orrective-action-plan-workbook-180818.xlsx | Bin 263686 -> 263686 bytes ...awards-audit-findings-workbook-180818.xlsx | Bin 1125822 -> 1125822 bytes .../federal-awards-workbook-180818.xlsx | Bin 1551168 -> 1551167 bytes .../notes-to-sefa-workbook-180818.xlsx | Bin 264387 -> 264387 bytes .../secondary-auditors-workbook-180818.xlsx | Bin 295039 -> 295040 bytes .../additional-eins-workbook-217653.xlsx | Bin 225786 -> 225787 bytes .../additional-ueis-workbook-217653.xlsx | Bin 226646 -> 226645 bytes .../audit-findings-text-workbook-217653.xlsx | Bin 263674 -> 263676 bytes ...orrective-action-plan-workbook-217653.xlsx | Bin 263688 -> 263687 bytes ...awards-audit-findings-workbook-217653.xlsx | Bin 1125825 -> 1125825 bytes .../federal-awards-workbook-217653.xlsx | Bin 1550645 -> 1550645 bytes .../notes-to-sefa-workbook-217653.xlsx | Bin 264926 -> 264926 bytes .../secondary-auditors-workbook-217653.xlsx | Bin 295041 -> 295041 bytes .../additional-eins-workbook-251020.xlsx | Bin 225120 -> 225121 bytes .../additional-ueis-workbook-251020.xlsx | Bin 225116 -> 225116 bytes .../audit-findings-text-workbook-251020.xlsx | Bin 266692 -> 266690 bytes ...orrective-action-plan-workbook-251020.xlsx | Bin 264878 -> 264877 bytes ...awards-audit-findings-workbook-251020.xlsx | Bin 1125688 -> 1125687 bytes .../federal-awards-workbook-251020.xlsx | Bin 1550374 -> 1550373 bytes .../notes-to-sefa-workbook-251020.xlsx | Bin 264624 -> 264624 bytes .../secondary-auditors-workbook-251020.xlsx | Bin 295036 -> 295036 bytes .../additional-eins-workbook-69688.xlsx | Bin 225121 -> 225121 bytes .../additional-ueis-workbook-69688.xlsx | Bin 225117 -> 225116 bytes .../audit-findings-text-workbook-69688.xlsx | Bin 264510 -> 264510 bytes ...corrective-action-plan-workbook-69688.xlsx | Bin 264168 -> 264168 bytes ...-awards-audit-findings-workbook-69688.xlsx | Bin 1125733 -> 1125733 bytes .../federal-awards-workbook-69688.xlsx | Bin 1550558 -> 1550558 bytes .../notes-to-sefa-workbook-69688.xlsx | Bin 264684 -> 264685 bytes .../secondary-auditors-workbook-69688.xlsx | Bin 295202 -> 295202 bytes .../historic_data_loader.py | 21 +- .../commands/historic_data_migrator.py | 21 +- .../commands/historic_workbook_generator.py | 6 +- .../commands/run_paginated_migration.py | 6 +- .../sac_general_lib/audit_information.py | 61 +++-- .../sac_general_lib/auditee_certification.py | 18 +- .../sac_general_lib/auditor_certification.py | 12 +- .../sac_general_lib/general_information.py | 108 ++++++--- .../sac_general_lib/report_id_generator.py | 13 +- .../sac_general_lib/utils.py | 24 +- .../test_additional_eins_xforms.py | 20 ++ .../test_audit_information_xforms.py | 125 +++++++++++ .../test_core_xforms.py | 212 +++++++++++++++--- .../test_excel_creation.py | 18 +- .../test_general_information_xforms.py | 212 ++++++++++++++++++ .../test_notes_to_sefa_xforms.py | 103 +++++++++ .../transforms/xform_string_to_bool.py | 11 +- .../transforms/xform_string_to_int.py | 5 +- .../transforms/xform_string_to_string.py | 8 +- .../workbooklib/additional_eins.py | 19 +- .../workbooklib/additional_ueis.py | 8 +- .../workbooklib/corrective_action_plan.py | 6 +- .../workbooklib/excel_creation_utils.py | 45 +++- .../workbooklib/federal_awards.py | 39 ++-- .../workbooklib/findings.py | 15 +- .../workbooklib/findings_text.py | 6 +- .../workbooklib/notes_to_sefa.py | 6 +- .../workbooklib/post_upload_utils.py | 54 ++--- .../workbooklib/secondary_auditors.py | 22 +- .../workbooklib/workbook_builder_loader.py | 2 +- 70 files changed, 960 insertions(+), 266 deletions(-) create mode 100644 backend/census_historical_migration/test_additional_eins_xforms.py create mode 100644 backend/census_historical_migration/test_audit_information_xforms.py create mode 100644 backend/census_historical_migration/test_general_information_xforms.py create mode 100644 backend/census_historical_migration/test_notes_to_sefa_xforms.py diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx index 1df43aeb3e4b21380888ba836cfc04d67b23c426..2b2d9e8c1b2bc41adb10a7a0e78fc6412498f443 100644 GIT binary patch delta 657 zcmaEOkN3enUfuw2W)=|!1_lm>P~M)2ylVx4RQP?Z?5$scf~Of77zC#q3{qGd$Vr%E$_SRHGF%IJLn#} znqe)z^xN$>4o_4auCQ;>5} zQR8FKS)#s213^1nDmdgn9Ebsgyru?N@hzZ#DcCgC$<<|qrWF)(a? z&1lNT3=E!TWsdF29E@LmK%92zP(~nT+AbZ+JeeOPvVCI;vn(@+HT_~b^E0r}t_!YrPn&cGlHj13rQV7$UO`6HvK zc7Qh{lL#|B_QKcqSU4W?VPshPL4ZL5rW!~$G>T5&kjtzNcEGJ%W@%>LP{!%np^TFJ zXhuv|$YYjfTs++-k68w+IvU8gI-JhT5MPv?ECBGL4OUu}r#+z3o5E0`G=_>rw; oTr+(`97qM*U5SJ7 zs}G3NE)~iM#7x_zLYXJ?gG9D(NMV*`2C=4JNN0Wq7TTG?e47z0G(C&?C0M9Ao7o>C zbTx zmrS?LW0nE)qkw#?Bk9Zx@kOaQ#rk?665Z0P4KbY?85tN>Ff%alBU{S2cKZ4}W;w7y p7eEGgPk)ogtSJxj+R}&85%NGy2`uPdE1zzX&#cCFJO^kT0|4_o_5%O_ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx index e80e31530479bf3855dee9c6f783d227aea8d192..50a5aa2b18ec52e3f5933ccb56b2f3621dc7e21a 100644 GIT binary patch delta 644 zcmcb2kN569Ufuw2W)=|!1_lm>aNeGYylXjuRQM!?ET*p$4{Oys@u{6nO#+~yf zEoZA~w{UW{O6_mU$k}B%?Tvbu{=c5wlJ&Gj%Es>Z!G-G0iN-hA@jLCzEmA6CQdl7z zCAsOvi^UZZd!~v6*R!Z!ZMRW+(i566amh55M7Cvls%qj(!n7~d{>o#_V7;;M=sczm zJ%`VFzsUWuIWaKoxyYGU^-{+TrMr)cn?L%n?fzb!)619ft5mmo24|~IarbP_f7X4c z@>SLj?Q7F--e?ECA~75RGHA{8wRy~PV1v(r`0dkQ(XBJ>PoWl&11ps>++Ij#0 delta 642 zcmcb6kN4g^Ufuw2W)=|!1_lm>m$7XVdDn6Rsc_!n-?P3=JgimkbZ+o5ndE2gL+}7iGYj$$t3C@NE z{TZPrlrA1ExL>_mBknKboJ;br8YUQrHMs`4O>FXOLO;NgQX18=ZJ=M{zID1`5?$|x`>;3;$U--7{ zrulz^XPVmCaa~{MKKO2atlJ^%!|VG`#8HAFd}hoXWdSw@hRv@SP1%@V#kMsoaco!O zVEpO>;Zh6EM{24NI~OQsvY~G%Uyld-$RQQ#!2A?lL!PATk41z#9B|kZ!D8Has zKRLfBRj(p9r+1=n_F)5_w*OuyZ714$@vEIpO#+~yfEoUolw{UW{%HQwBo6O$c z;A4(m^8KnZ=hS1`m)UFh_8xcOJ$CiQEj_Kr+i!NB&~C`^dr_h@X|eaCh5UZOUWWzi z_n685>QIol-QavFY;qgtjNh#@VkcbF@%S%!OEQ5sp_|XT>i{>Oxqf27$MbEGk#mxB z#I76g992k*F64@Tl$s*4T zVoh)4VR;4?%Hm~VW&#Tp@Uy%G3&jYq_%nixo4!tv#hMcy3)4RdvY3IzwS`!unJzO< z{>UggJywW?2P~Ka6l50RV3>YEh(%=jJRufAd7w?5&W=J{# zY2EJry6XT?sTfc@H;UGx>HfkjQeeO32(w5tMo;e$W|0H)R|5IWiTo_n8~9nI`O%D= U{t=|wcRH^KiyGTpK^CB)06L7?RsaA1 delta 671 zcmdnKLSW|#0p0*_W)=|!1_lm>AF*u{dDqqhsqmN|`!0L~3Z72h1jlDpf@*Y3V6 zYRj=RR{7qa)CuPb=5^5}+{IoH`2-Zirnuw)eY!nl?zQstM#JGOk$qpR;X z*9ZS{ek)jXOlyP3DUBcl?%?fGlP6z#5&F`ub~{H8+nWnX{)`pJ9v%{^lKz(yF=^@? zr`}dnF_&v@+;0OfK6v^^seH~ZsdZgnGryHHo>lV` zC@IqL(JuOa^J{ml|E?MAm*3s0m-sv-ly|*LrShrU z+K(d@vVVNd|HO?FTALp*+OaVM!>d`5W4j^;;|Cw+pRsN2BB6{x%(Pu3l-X?yNMw5~ zCyP8Yh&8>QhvgYqD1(=UnF%bE$ItQ-EEFZc;?D>&Zu%NQ7Hdv;JWT%}$YKT-*A!xr zX1dBa`8}iP^k^X#9y00*c6xbtK zAb#xhHenVyMWE&3tK*9FwgAoR1sX2}GZ#oVFedY}Ot0f-k)HlQ80ZO*L*Ik+`cLN; NVNqk7FUSH^1pttS^1=WB diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx index 14ed7510faae31b3dcd4571531c0ef146dd14059..010f4cd27d487029fa2419c93df26f45894fa18a 100644 GIT binary patch delta 667 zcmbODBhllylVx4RCwWut88C^f~Of77zC#$vayKN_f9#x>X1kxsyG&e^CcRValUt17d4|w~>-VqJhldUe zD0ntMVzgsp1~QtJIkqcvF#hrZaoQC^8G)E-yFw`Qv=tzc?K}T6%QJ&mQ*X081B)DC zWMN_gv8JD7W_bw~+RDP>&j=P`XJfJE1j<3eeY!UriJa*Lj1~7Tp zurM(2BU{V3ZTdrY7CErnR>u|TZ2?O5GBGenp(xrtU5H?eIKdDjX8sqlLV@gA1379ZuU6E8kMK`-#UB7KaP&TeMOpzDV97zb8~={XYi3m;9v;CnQ%NP|{rN zaZtzSZkEqEuf^d@e}B2FWFWP{x!acYfn4X7+_u{JvG%{#J6X>tyZ`U-jOA;xDpXIcJ#kw4v8zJ%kFWWkxM9J= z3J#ym4;k&)n1LbGti-WhiG%T%4~Ww)AIb>COxxu{nWwE_ejD4ieaByBd1er6>MfRM zV3EU&EKE!w*7Ot1EHA-ATUc2989@e2XJccr=7b0QbT2j*GkFkqX3QLA0X7DPIqD1y z!oc`|fd@3pE!l8`Q z^+OpY`2)NenM9Z&(GGG_08o~3&Ga0078$S;yMcVGa||pD@kOaQ#rk?65=~F|&tpfe zZvfMW4GRMUKeDxq+owNZXORON#K-|Oc+YfM4i-%?-xI`NF+G)oMUCwTD^Mo`03GS? AvH$=8 diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx index 176a55a26102bb621638cf6cdac1c8e84a2f062b..e13eaa83580241182bb2594c95f3a2222f05b4ad 100644 GIT binary patch delta 764 zcmbPm)1~jG3vYloGm8iV0|N&`0dLPl-nI2WD*T1vJe@BdpRce1*M$RtF zX>ZiK^#AqbmMI@EePNvM{N6C2xzKdiw(d(ma?6xH2`S7Hwvz3B@?uiOi{{HyE)}rU zuRq&vr!=J}_kd#NYR|(gXZ|{!(eua_Tk@ahEzbtF4X*97t`At-XHPfS@gx248l7{7 zX5HBmZAk)~^>dFr%-vr-bNcz&dXwJqq)pxAHmhOQuK-{1pPz4N?a*Fz`ptW$TcNWQ zUcM=u9XmDr*9r)j_c8u`B(<_liu7H9l()Csr-C^ zYY11}-?hJ(P{L~SBSt$meo$xycW6zy?{yyL@M z#M{%Z9m)vAOhC*G#4JF}3dC$c%nrmHK+FlmTtLhX#5_RE3&ecewL|&Sw}VZX$|JCS zDvuzq2#B|Rse+&?Gl(_)tg_%Uu+TOY!QD(?p%Qh$k6@t;4Z#pbW}p+N@7ENx=7h)W zbY?9yA<&8ZF4+r}dR(QB2Nik`^nDLR5`Z0j^Z=@|gCw+Lwf delta 755 zcmeCX=`!J_3vYloGm8iV0|N)cpV+pEyld-$RCs2VRl_%+;Aut%20u{O`>+8|+kdZXjz@1TKdwk?o0J;d14u?Hx@d23sxL^cuI&(_7dz0@wQysoR2-^v-kX3bG} z@uYBe%%ts){w=KZc{S&$>#4}74{hs@mi##WEY6d{3Cat z9jg$$BKqgo_Fvp6p|$w|qa7Rb-`KWhMUL%?9E_~K%nWgD?V6#CK+FWh%s|Wn#H>Kf z2E^<@%mKulK+FZi+(66&#JoVvw_P)oKYcscgvmSt+b8n~@``|X+ZQVcsxpIE(@!f4 zJ_8GFQ4!qD1Qse%7yJkoO4ASwVPpnI-t@hig4UeySe(wNC1?f~x6%@nX1dBa`8}iP z^c*cg9+1nYSAzt_I3^o#Nbv`FGct)VLlOhXx`RMDX1P$t$%Y&v(?4qo3Mv9c!!=H^ z&Albe$iS|{$shqt8!*t&*uf(JR%)p&C=?JQ^ZO{ZIV*n&?1783D diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx index c8466df89885d7e4b75e4e37da8946d08ec17dcb..bbb5a34b781a38479459e4ff507d9a65890e6d9d 100644 GIT binary patch delta 888 zcmex#DemK?INktnW)=|!1_lm>6yBbRylXjuRQTEUeV@NhJgiZFqR%@~?d#Iy<-Ba% zId9T(w(@ohCughF{&Y!zkC(n=wrjtaynyX-R!Oe-(vP>_csx;c zFmW%j5<7WOweUXsa?Pa=8RBO1*E(oO-fnQd6gIhybH?x18FMFI)A9H(c}p^ZH=&!) zy6XTpU$}l^!N>D$k&$zf>vP1e8}J-eNQ=MKarpM-$EWwy+$)(_l5P+@)$|g}l`HI~ zu}}PyqYwISv^xJ!Z$a!*&Pjf9+i$v7ithgXOEFkg+x1+eUd7q#My{Io)UWmb)4uR^ z+fDQT3ez;RwoS}dym^=RBU6CrpI@_oF`K}5DNma5D*IkvB>s~p`r>em@|2MwqLOo<7ES}+MhU!ZGYk{{v-q} zetL%ZMeez;baOuv6nf!pCgIK*Ayh#(EXGImpHXeG{Hh%NM>e zgpuKAK0BK6*QOs_4m2L*<<}r{_RbIon=}3W3~^R|G;^jKt`L`I+&JBLg}4k@OAe54 z#Wi1?A-*Uzr&wPPM55^lSJY3~)5*-h@QH_kfgjmxjGw0;Tp=ze4{{)rlk1%mKuKp7 j1_mh&#`S2dDn6Rsqnh_e_wu^cvz!eW1IIepIN89{H1xs zZrnI~BTHIL{n#~+U*C#1nZ3Qi#~i=r|7CT_>9;)ZR@Z#n>+H|lb~WR+p2u6WlM_#P zHZ17R2tA>6@o2&Q>eU)?e;Maol7H23;PW1a?j=1+tdk8&yA3V1d*i0OyS#n#fs6xU zx@95_sXXg+7H)rdzRfc7gkil2w<(XaS;x$7X~T-@s+yV8{b%bkLR*Lf9u9_ZV^IR#lQY+%@H6uB_d+OKw|E<38 zZP`up{|28-^s~ZFe4Q)u-TYYh0EiRnpMeaY3-K!3uF{#!b_GR*1`h z`Po3e7598`hWMh?oML@F5Q*$JkbR2!341!385lnCFfj1LJO-p27{5$EutHo8Y|ulH c!K0P#*qk^lez diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx index a2110280d0503e255691ebab776a3419280d34f0..d8aeef8e2706cc0155f067ecd1df8bb9ddfd01f8 100644 GIT binary patch delta 487 zcmZpgD$q1lfH%OKnMH(wfq{degtuoR?^;1170&2a7Z!&v(gO53W&Hu~llHs>F z?^f4*-0SSm+jceMwjRe@vy&4~a5gOH&j>xCbn$4x{p!^kaeo=-Tv9J}I3XF`>>A`Y zvB`7BuA?(zrnu>QeSdj7GQlLFTg;m4fN?#a`LqkSKcpX?qw_>!R;yH-qIB2HZt22` z`nsyQ)BR`bO?YLZw#g_eqra<4f4TV1&o{yf!q@0+-p`#8R@k&)^I_j2-+Nte*VXx? zocpBoR7bbstoK2AmSp+KuigLqW>jB(GrL~mb4AuRu~bE`y7WCK5??Rf*Dp^EFJ(_B)h%< delta 487 zcmZpgD$q1lfH%OKnMH(wfq{d8DXwiI?^;116`pfq`O|Me!PATk41&`GIaoyMdnX?B zJ8Zzy7N7dcw)E08F4stX1wRGFK+Yq&UVGLq%3OS8#ogPNr>&K4Vm|)k|DT&-Q$Ak$ z!Z_dgygCLMj$xjX_$}cF^ zPtGq&)vL(O>795m>#zY&Tl~@|^G^6(lJj|W+RI;>N9@Lpvp2G&#ng{o^Z50xc$3-N z8+^?1YyMwWmkhtfdAGXe+g@jX-nOe5xAi#QZadj|g0o?Pe@5sDrHe;DT&P~H5%*WH ze$FNNR~-|Kw=s4v=}}^xY*5;5xKpDyZpypM+czJ`I3T85Cen~Dlc%$A`@{2XmXRk6 zO}I^YoXtdx#mygm`1a+8k^X$MScR*ZJ}IeNF35>~jq}qldcJY(gLM(HY4ze4);wZy zOmDyaWY!)r{@qpcPt1AhdMYyN!>s8|>sht$t6$gb|2O->w`GwN?2r>L9FRq6)exdLZ2&Ga+ttEN!2VL!9rm*EFp{_!>6yUWwGXjN6Pe% zwJc^}ajiNQX{IZTlRq+wPLHW$;QiKJo<*AR&h*H778x6m z-tF$MyAA*~h%qrRaKkhL=?2C>YFxkBXIt=J8IA~z>m>$%?qQ*A27AV620L%paO#lD@ delta 723 zcmey{AoQ<6h&RBSnMH(wfq{d8HLh(U@7j7G75-`&8}~P$;Aut%20u}E|F8j1+y7Hns$VbB746y;=P<{?X^~Jz^puKd*Ut-&thl>>xo)&M z3wz(k-#<34y>g2CDU%)hbHjl4LgQKY+!p>ZvlOUg6u8RR+jG<9hUT4}2`awlH?Zs1 zUv0mm{G=x|VZxGWDv4st>P*$tmxisrRNq!7n8A8uv7@(O#j%HngxG}t=R{1JDwDiD z=&(VG^TvY9ZZh}g$;D55dik8&q_D(GSE4vY1152wt(j=wSIZlI^q0x+-3?X7z5*p* zcKDtEHyMv&pt*VM9DbHZa~ z`iEK;GqAX39g8&6RmRC57)7T?*Rk+`1+zecVjPnVIi&alycwB9m?4o1blCI_KsjdN zP{!%{p^TE+$g)7kg#Qj)b8#UH1H(>X1_lXWoWnpvBinSLdKPuC^K9x_q#5r{kEmyn zu>on??*6*#08phE69WS`Ob3u|VEkLo!Vq7Snp3Q=2O`mIo4x^L(6i}h>RIH#2EPUJ tnM12trWaPTNMpF|_V083D!{b(66ijl^WdO?(Q|rW1B)8lyjq|P0{}a~@Kyi- diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx index 6153bf7a4736645c9e3e17d16284f8d73d3662c2..b3a568a92f523b125fea1c86d287f819473caa19 100644 GIT binary patch delta 501 zcmaESkN5FCUfuw2W)=|!1_lm>hrB%#dDqqhsc^qn)9!r(3Z7YaYM86>lXPEOJnvT4e4Oj-Z`yV=VYZpaTeFiBPk1&g=+6i}p>*+B!TrkB8gYLa z>*rijFLe;;S$jZ9bFs%k4WGL?K6+k@)0h7Ka#qDaYJ+39F6#ryc4_Yyxj!~1dZiU{ zeD(QSJOAs?BS(8J_SnCVTmJO&F5V?ky`GU0x?9~{=jtC9-}(81NI|%c-sb(>8DfP^ zOEw?&E%Lqh>G<-j*;AT2ZJz3!tvot?((_M=e+_qO)qT0&vd#GGzJKk3M}jv_jajEE zTz+iz0-Xof_fL&S39ZeK812}Yf#KDx%&}dWgYml$h|?|`$_T_v+hs$UC-Z|uwr@^h zmSqO9re98Heg+oWlfitO5iB$-i}@v3s4bh>A0l)khuN9~s2m&*)1`Bn&A}=|bD8bH ojA^;d&S1tJ7(*$K*#ayQo5yShQIp4P1!i2%V|E2Ir1P0Q05D3}!vFvP delta 501 zcmaESkN5FCUfuw2W)=|!1_llW?YOpyyld-$RJeNZ?BH)e!PATk41z#9B|kZ!D8Has zKRLfBRj(p9r+1>S|6v23w*RNDRKH#(>FN@zui&Sk7|3}<*K5z(MLz?QuDrW{dD>d( zro(+7fB)FHHslodQ>J|^&kX~*3r%O7b6faF?WS-gqrg?U-kzHtHw^FWN>J%Nzkyl5 z{_65OiXxS9O^%bICbf9Z_;qxK%+x4dum7UEA{U4zw6p2DCP+3LdrSEI*`DYpR@Cv< z>1*x$uQkt(^-Aoqx8Hm2^zya6Dr*ldS+_+*}^Jn~;HAmsa zgTl#kCYjg$zhI+QI#JvvwenQt;cHEWg~h)Och+fro!@%wW!dexM4d}}aw0UZl`ncA zer)oBbq~((zj_}fv^GCtv}0oihF7yP$9827#_vANI&p37vZ0JX%(PuLlzB2gNM!rw z6lPgw5NrD7bmnJZp*wxpd4H diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx index 0f7797140d2b91cccd98cf71a99cf7a46d1d784c..8dbda7eda4dc1c943e35e9d0f77e47cd257dacbf 100644 GIT binary patch delta 692 zcmcbAkN4g^Ufuw2W)=|!1_lm>N4z}~dDqqhsqls2_qx6U1y3_FFbD$al>Fp?qWpql z{p9?jRK1GaoZgAP*@q2y+Wvc;w4FG&M0SSB^xnB<5;_|;&fa*{Oy}Q$YaYMeKTpZM zwNaLBm&g0n?n0}kil1Vw;X9YSfbVgZ$=+V4pSfiUpO_R@F-u8yKiS}UXJ>-SQj<3a z>%G4!e&f{#LM!vAw-Og(CO zEU%FFZ~)KMH{Q~P71dQWGpC=omiCNljSS9K4RP?)Gr8>g&*GbDjOgpjZ{9P#3Z13! z;zi-?xl^yddbh9k#H>kEL(fIVeVn%5mFw6&^=ti_|F|z$+2XMP42+MU6CgApt=EsOajSg0wR*`E<)-1I9s%+{RnXqYaM%WMV~56oqj zX1c;S`6HvKc7Qh{lL#|B;=*+* zO5kT=5Eqm{Q^Xg_IDJDdv&eLZP)642{CUhgV8_YjF-tQpnQoQGECc380{K=)(wQ0J zi&Arn_4Pm`vP(dg{QK|odL1y`EN5n5;DFjH0Pzx^X_U8rzW^pm7WU-az>1 delta 679 zcmcb2kN5sPUfuw2W)=|!1_llW-MF@iyld-$RQTzWy*%H5f~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBnS%(dH+Txc!nRmj^D?#n+(&Xj5Y}`3-(sH))b_*wGtJMCsjGVo# zq`gu9+W(i6d$ixSytCc+>)yiUY=^V9f$Hn?`nvObV(=k|V)`(tyWpI8yk zSts7Z6L^oFIofMcv&Uxt+0)B?#XU-;gd-(&x466RoqoCdPGOGr4y~}|H}|uethR8x zaPgsz)AyhnWdSo zGEV-$D9Rt;&B!Fe43D|#6G3uf9Fq+>q^6(9W#&-?>HmIxGE)LS6N9**1cL-H9$}!N zkw27ix^^g|-nI2WD*V$NL9MSq!PATk41z#9B|kZ!D8Has zKRLfBRj(p9r+4B(zrzMRZSkqE_Lg4qadwH-SMXC%4CFkb>$OLBQRdCTpDOA^?upL^ut+m|0x*^ObiU%Fmr%(1EcA5 zK~@$iuu*!fK>OvUd$F>}f%(}WzRL7=Ru)aLdzL{G JUE%3WrfH%OKnMH(wfq{d;Dz0rJ@7j7G6~16{qT)B8;Aut%20w8C)?ovlw)mw_=AH1{b-?GZ+o5ndE2gL+}7iGYj$$t3C@NE{TZPrlrA1ExL>_mBknI_ z{hUkkuNo#8hc&qdxlL^HoU!ZZj5$-?^u4~nyd9ZflF%(?&2_+-FMQgC+aJ;o&(V1z zF{@RoO;NgQX18=ZJ=M{zID1`5E^LqgI(ew>=NS2AkpAlr-^n7L(YfgAPOyA7RVg?p}!^|Sh zbd_=P2S(B9ax5%7U_o6Lpr9DXWJ3-q?Er5^CJ|;xbONmlPZQp|>&j9l1`i1p1_@w1 z!azeK*YtiC7Im-#b^;Bs5&6x`P?4iwTvC~nS`3O|bfdPrzwSB!)Gx-wz`zYN2S_(C znoSpAWsw3KrOV19%_u+Jla)md%+CVynN=BCraLmSNMpEe=|kxVd0+}lU}0boMln)w P`Z-n>HMT@%pbP^5-&)}j diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx index 6ca3018bc89eeab2f4c3182954846658ced3c724..62eba6f6a848c7ccc94f99fe6e37806da15b0096 100644 GIT binary patch delta 469 zcmZqc5oqfX;0^F*W)WdvVBlbQ!rL>EcP$5y3SY2Xfob9)o%+*0uTH!9%kqfbxN-JI zmb8@mv1=Z`z7=mWdwYYAId;ka>*}0SZ*ktOtobCJD`A(Od_N z`ONhfZhv^b&2#2l=N0MU_4mI=#{2BsA=v)?`Q6uBYNEC`hO9Q6xb&h^b;y-U%eW`! zoP!U=ZrpR`pWcGVBb-g=_{>j+R%*-5UzMv=JWX6PJ@cUUJ`uwkzmik_zSz&0oBpcy zFMpEU;w0^jG2Gt@-1ji0aPI%7wx0nd5W;81%uyC#V_=xG`3<898#6FKnw2@WD|0aZ z_F;a?+taQX$_T_v+Z98ZIhTP%w(I_8mS+aBru+Y6eg+n@_|MG51QxPoWO)e|(qv-s zX9O8Hy_%WD8XnNo_c61WgN1)Ev)F+dmMkpJV7>J)#!(g)3y{b(Ru(g`j3q0J6_`=R N%Hj%U>|-Ojo%@vYm;&+hx# zZs;Ew`8jCKd)I?`J>rdP9$eplQXeG{!e_?JQ5IlhV3@P{4WkJgGcZ7!l{vO6b1?q) zVK#_sYgY_q1Y)M`ilNM$%RnOAb$>Iv9fpo0BXmz+W-In diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx index 1c4d333cabc64594893ef8c7514f5007efb6c332..71fe6398a1f1fe9f975c690706e63cce6c17a203 100644 GIT binary patch delta 570 zcmdmY+hyNv7v2DGW)=|!1_lm>PrN-7dDqqhsqo``Q>(rJ1y3_FFbD$al>Fp?qWpql z{p9?jRK1GaoZg8Cvkn{Zw8by|F;BzKi(Bn$YVvYkHtw7^X*pYYyOWc%Rce1*M$RtF zX>ZiK^#AqbmaMm}WsLKiKTE7Qx1@ue56FYTSpnS0pYTuAa4tT^}Zl`4CfLqrd zVN+Xt{p5I*u-g29(TE%>j$U=?2<@=3wD0Z9zLQW3{%RGg$9O7{go# gDpIQ>XaSbmp(AJpQKKVh1!m~!3c7+BWx9eM0EJcNI{*Lx delta 570 zcmdmY+hyNv7v2DGW)=|!1_llWhq$(hyld-$RQPQ9TMNDb1y3_FFbD$al>Fp?qWpql z{p9?jRK1GaoZg8Cy^a_Nw8f{M+&g#4RXHbf@uQQQ5*$B@O}cqE>2gcTlDpftuibrD z)Rtps%D?=5Gbh|)$*ei@`~|aHmSL{j^{<|@Hl(sIQed29x!|(q{Ej-Vayu*E%ck!X z>oYzt{LaDpT+~7(*EDlRYh*<4n*Mf#G%4~K5FJNr*>!CfC= z6S``xX2iSC~8-W5-Dr~kQC zA+|!Q?(ghhOekTs`2nLH8#6GpniV;=D{?Th`Z7DkwY6)8G6FFZ5HkZY3lOscF&hxG z12G2>a{@6J5OV`D4-oSLG2eF0Q2yyVz$P5#71(~5SCCf(#M^#ZQBai`#G3wFMerF| z=!vS}ZYHqMA`QWhV4-Q6f+36`$4q~#C1}k7i^J&#+Jfd_;Vf-IJ1}FlwxBau??)KJ iTn8#rt0QOumfE2sXa-TEBWML?=;;c&f*EDHf*t@aSG+wFdDn6RsqiabQ{I1_cv!Q(%!^&^?9$}rylmV# zZ_;wM@^;H6XRFlywv3!zmeby-cj^D@$t_xMm%d}RYrmJgKOPDw%a7hAFTAku1<4Z4CU)npHGxxB)xv*$4 zht2bXuR?6n|8r(cJ!*NZtiF)<@C5#+Z@i@oE2^t%W==mJCGKh2yJlU_G^+)R?5acQP2jFUeyit-0|Gct)V!;{MNg&;Xm zj_Dii#YCrHULh{12n?F=679HGJ70=0>C+z8DW?=Zl!@$4~b1sl>U_3azf0ei# u*q{wSgBZ)FpIIfY33g=Kyz8&D85tOU1LH~7|#M97)v`!#9LkFWS2EcrEoy~XCVmVrl*_ap;u;q6j4BUfHsRWg5HJWCYgw~T@X z3^q>-QaSf>)|=^^^zvI6UB9wW=JLjf9XkZuw=X|FJ@w}jX{M8oirHCfTRv%={FPQW z(b~6`*Zk-&o8P+|s)W@!tSs+r^8OU{=G|@fYfmEMRBXJt?tKb6GFkln=GX53Lo?RA z`*zp9r*cxL?s}I>^;4gYRS24x{`s~288=EegwKqbqb$J2u=ypU85=V&M4FX2wkvTk z_WCkg#I?0=3S|UhCLm@8Viq7~1!6WJW(Q&pAm#*OE+FOxVjdvo1!6uR<_BT{AQl8- zAs`k8Vv+5eLPg)a1ev^jp}iO{8`zGGu43Cax{5yu0gKO=CH|2K%<7pd9>xq}O@F#T z{25s2+(Plq5TTZ(;(x(HEz88C89@%8{%E1B=J65SM1U$~gH0qbPrX zHzSh>GdvYcUjUL5&7p3MD>+69? zbYCgzC+z8DW?=Zl!@$4~^AeD5U_3OvZ6!~ XeghpPjH0Pxy6|doHMT3u#eu>AJsvZV diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx index cc411fd8df170e96e557c46de62a4e29e414eb96..9100c4b557a9543efe91630342a241d17da3d8c5 100644 GIT binary patch delta 500 zcmX>+QQ+`I0p0*_W)=|!1_lm>ue?1IdDqqhsqhvvas6*V!PATk41z#9B|kZ!D8Has zKRLfBRj(p9r+4DPtiuL8ZShN=%sb(?iO=WNX)k|i9Z+o5ndE2gL+}7iGYj$$t3C@NE{TZPrlrA1ExL>_mBknI_ z{hUkkuNo#8Z)59T(xb#W*`T!B&{DfMZpypM+czJ`I3T85Cen~DvrcE>_J`-&EF(`C znsA%)IGc$Wi<>|C@a@YFBmMbiu?kl+eNs}lT#ys}8t129w0z^*2g@Snrq_#KSo4U* zQN4ZklSzBT_-|KDKQZU2>#4}753{B>-RC@ZPyM=9|G(82zAd|H{@>u)$16Fyr>XV0d)@#EKWi+v~Sb#+QQ+`I0p0*_W)=|!1_llWm$%Zi#$OWPa?QFWP4W*P~~8r3)*nt7>LW_m7tLjB53a%~TC>@YK6<+4Y}EiRc}X)t}$IXL@zj!tuh% z$7(xO)*n5*?WSt!xlc+@b#yBZzh{Z(K6X$2dcW4c)fX)8zPnv7@p;NrUj3u9ILqpj z_i()8s{0%LmlY+nHa}vtV`Bz}SFm4=ac%84BN&00Y5UCx=BG&EcA?p#h(#m+H?yx7HbYzG)%8%V=)H{Z)Ib# s12ev{u{eYE+OR_zP3$ZdV3ED-EM{QFH+B{)FvFOG#TCq`=3wyv0CCvKcmMzZ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx index bad6b0ee9f2144393ff51cf754a2117029ae81bf..b957a845d9f3293d573a4f093481a4be37e99438 100644 GIT binary patch delta 742 zcmez0Ak@$(#2etv%p$_Tz`()qgSTfQ@7j7G74EW=X~%b<;Aut%20pQH`>=t)@%^V>R9AaOtGBJu@ABtrVm-Qj!kukNm)o8!xx3wbZM6Bt zcE`A)|M&mCb6Qg7^j$8#=KY1>=0ww5+qj)}`)*P%Vp4cv8zs8Qqr~vew|iP$;rp5E zmw#D)OYuq1+Jj1(i#-mi_}oqM>GN7FzV!E}w>kz=8(h0(SszHY8+*UV{jojKPppXN ztP}6y3A{(o9PPE(V{gCr-09`M;vS_^lf9&Lx466RUB66x=iwV(J2Xx`elwqGR_Gjs z7Y_<1&zZFS(Z7XtKC9+Dbv+ds^^l;GO@h|!LX9~5BO7W%hO_%Jf8_#iM{p@2oA*_~s%I|t)+ALgID zJ?)C2j6lo;#LU|jLs>pZfkd~96tG-j0kdA0u*folSkt*HSe}7}zErT}FoA`Vt64sR zg~DrCLKvBWwoPAG%VNz5kEiLMYFW&{;@WjA(oEMFCx2uVogQ1q!UGn}0SSt8Og9Jx ziq5NJ5mW?P6uyqNd*fayW`^8X`V11l7>0p{M$u5l>4u?s6Kx##%Yt^#|f?aD@&mzrue|l6siwu}w3FKS-FK1zhFG|fR*4G1(0p5&E zBFvCP2Xf2tWhdL5m>3vb*%=u4VGaS(4U8|QpQ~q)0~_=nWN`R&?gkc3d61^tzt8on c0F&oSpu>Rff`bM|@9Dt}ENW~EYJoBg0A@21jQ{`u delta 730 zcmZo@6#CyF#2etv%p$_Tz`()a5!W`6cWph83J=~Cdi@(v@H8U>gCLMj$xjX_$}cF^ zPtGq&)vL(O>795mui1d7Eq>`K**OlU6&J;Xecj?Yxou14c8$Vl6Q;mNoWH(tuUR^G z+u^>C|9@^?JLQ(=-Rhced!7Ax+pcEZ*7JC~?TyP56^9GzTeOx;e34Y~VsEI({C^yN z^)LBLT~1i&A5hX<>~T=V=T1_J_|l8rmui2NiAnI@Sah_HsUze-uj&`GKQ|53xh>n$ z7PcjNwAVa)Pwo?QI1r<0d)FR|)fqt!d@*5XC?!Y{kNv)QJ4NA&dRH}9Edh0alU z`KI9Xx|7-eHmv@>=2XQr^(OV5D~?8W^*(w1{p8ohTL0Cy?9JpmYu^)js5?re=F_k>4TNDyM;0WF%u9oZ+8o2IU~gkRJL8TfaMAcnDwTFMV1-F zn$AF4WM)WNp?1ZrhwFJPH|v5rM# zx^_K_AXwPG9%#e8>CyEpGGKlckZ<*`oP{C2C^e^8Uk^kEcr!AIFhdd=$iCytPPREQ zF)+BYGcfSOECbRFjL)Z^uV;}18}tEWaM*O71{O`QBX9pc*RKLhk}p{p7=%$Yc})*# NU{Pb6Ukj9B007lu5#j&< diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-eins-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-eins-workbook-217653.xlsx index d71fd578b47f84ce70de519fe730ab17c1c78fd8..e2d240963791a43e4036866a3c8e4ed02c430d68 100644 GIT binary patch delta 677 zcmezMnD_T%Ufuw2W)=|!1_llWe!iZGyld-$RJfy-^v3T%!PATk41z#9B|kZ!D8Has zKRLfBRj(p9r+4DPtiuL8ZShN=%sbJyJ8y=`^r`dABy=`xoW1d?nT}u2+KFG@K2OQL zwNaL9U&R0A{ziIRChz+EVdK3w-XEO5^+ z+j@(o_E+T;({otGJSAPZ+!9}6g z%M4;o&&pzc1{Mm=X1>h`7Bb9behC(m&13e52<7K9TXO=HLn31O=6q%|u+lgA%+gHP z7$<*Z6rC{)Gc6^W)|mQm_D(9S!8;A0kfbyNb7qi-`H=0ObpxPB^V@taR>tq zjY8A=3z*fxdUpc#+SsKtGgRd07nfA#q!xo>H^7^bNrV}mD#EwBzwSB!)Gx-wz`zYN z2S_(C#!VL}WR`+BqmWse(S5pSA+sEqp9SPItL8FKcg$s$)q`0<;yld-$R5-K8DY0)r!PATk41z#9B|kZ!D8Has zKRLfBRj(p9r+4B(zrzMRZSkqE_Lg4CaqZd_=P<{?X^~Jz^puL|t;;%v7MJDwM{mpH z;FEd$|8sh@=GN)ASZg@XB`;umoV6iWeCfB_PaK}8I$UAjqLrq6vG?IQ*3}s`ZMGHl zOZ&6>Cm4q{xCXgRZ1SA(>*$O*Q{D8v{!8A9OfX647PIa;V9Xaj&EUq5^uu#>o(Rlt zef8V!_0coE(uEcEbyahx`_I;!^vXmnG;_-pM$w%CzT!VGUzk=9taEF{g>g&qG{bzHivwewO&=csQA0#&N{6x@vYZprd|JYc*gQID=HPuV>W&5 z{iw9U>&MsptKul3wfQll9UC(+yqZ-wwySV3{_p{D+T}wTftYE#d?>S~5J+UZcRI5y zGl(@kJB#@lSSTc$`8Fe1$S9ZjC0IxcD&3=G^bbAWUMW9)RnLS`w5{e?jL z-KKjLGRuMa*&x2k^!7q#O|W~GK9r7-2PUBe76t}k6hj22pD$!qV@t?q1_}cJ0Xymk diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-ueis-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-ueis-workbook-217653.xlsx index e3e97832807063aa581eb630a03efe8529505585..f35bbbd08986bb08390cd984abd56a963014e97b 100644 GIT binary patch delta 634 zcmcciiudX(Ufuw2W)=|!1_llW0luDzylXjuRQQUU?>~Q?cv!Q(&s$RM>(b=qylmV# zZ_;wM@^%X+XRFlywv3!zmeby-cj^D@$t_xsm%e1SYrmJgfbDTsNv`i{=jxPD^6$MbEGk#mxB>cy@b@Elc0i@()z`1a+;r}xy{E16i5ZV)`x^b*UJE9|DR zPyCak5BhGjI{!~^LF`h_Nq%zMZ@N~B?*9EtF<4dG^<1Q0#o6mdtjX_%U%USoeNlD$ z&Fp%C(;}v~Pkfy#@~EwXaRpc1U*B3rlwb&-88b&&fQ^A+^J_*^HfCUmG%Is#SLR^+ z?!zp|*V8T=$_T_v+hs$UJ4Hbv+gD~W%QAyl(@*9yKLZPG%45F42o~xsV15Y}sw`yo zX9O8H{d5ttH6o;^^A|IlDS!eVX7L<#1_ohZtiV76;}yoq9~nibR~0k!fKBKuW|ju> zmliXtgDp7^;`2{`4dlxM-4Wj7_L2W969dCpJ_ZI}m@z=QfpPY9!xCm0u%>_#pr+l^ zb4r+{!2C`i-(g7}Gebp=etCXTc2a(Rwq8YUPJlNflL#{;nScym9ap5c1!!0=&;ltG m!|Mx}r_U{5mY)8<1n5vufbf+9?Z}?4Rm!Z!w!4TKs2TvqgV;0x delta 674 zcmccmiuc+pUfuw2W)=|!1_lm>_Pg z8)t81NlU38yXNuhTk$5dw>S8hnG@xsZ6Ry$SlkETW2s~M$NPAr-xw&z*7lH4&nwd?)=R$utG z?562|gJ+uB*>PRfdN&UDepH$u`sbJVPgaz02%i}f|O z^ZJkYFfuIpAiy92QwF3P8pSv!8*)fZ-w#v?w*Fi(voxc?^jF2q>hd6+J#HWQzcMi} zoaJL+;6>KSIA^*+39}4XlYa?N)1K+sCCpMWAWi@N`@CMq$iT3inSp^HMbpweW`>F! z{qp>x?4E%2vXfH%OKnMH(wfq{cTmak_b@7j7G72cY(efc+_;Aut%20w8C)?ovlw)mw_=AH1{wMo)^`qX)55;_|;&fa*{OvkTh?Zhu{pQq&B z+9=Dl%j5rgf1ydYIPX^0e4Oj-Z`yV=VYV5^+iNEqPjEIY;Liv>v1oDcqjjId0)76< z)z7&!KdXO&@iwOJB|S>4$_AyzhL(!G^CrGKZ7zHu;{coPHj##O8NE2C{KNZsCmUI} z2b^EG@BQX|GinNz`QD%RUw`>i=3Ta+wa1oBapQg)82J43N2PKZDb8@$%2|Jd9X59e zv&hP>o4ol`m`&cQ+@NPV`a$O|HHFt`CT)0>obp%e)qaND^jER9%!d7e8>dFx>)ze2 z{YWyv>&NHqpQKSjYx4s}J2qwozMf`9j_ryZjGuf!oOZELMj&R|E*8qnz6>O?UEw#g zJTr(j-RU3mGq8~Me`aPTu#h1m%S*73EE9`ABQr4gr{^=XSaZVTVftof7BjH;8)g=1 zrmKvT-!qC%mt$e!0qfUg0Sbz7Og7+<(hl%uWD;S9L?_U?@HFAQyRIx{V(^eqVUPgE zBMdY&a!>DPVNnMguoGy2jp%P?hKd~h;*!do)M8K!qZ_r|{dLy?pnfqX1_o}JIY7FB z(R{i9D~lAwEvzikj0)2|Sy|-3{45}!S&flpx+5ctG=}SzK9r7-2d1zD76t}k6eESE PpJQcFV@qNN$}j)`Qn%TC delta 673 zcmeygCLMj$xjX_$}cF^ zPtGq&)vL(O>797c>xh9sTYT!vy>pjblyfo{KRUT7!SSQmq?>n>F1MvDxx3wb?e4px zwj4WC{^jqRIpJ1CW?RGg#19fHj_#NlbKO1jkD0YVC8xky#<_j2O20(jwLLIdnth|W z{`xcL_kv!>vdKqA|felyE6 zgILpD{xLrT3+eu6W@Z8l88foH1PjSCvG_AG0|S40Av23LCp;FWZ)IjN1B<_7W|3yP z%sBZYqv&)678V|`eti}eX-1Lh-YhKY@<1!X(}ef#y0Vms!9zlYK>`?aFwoG*Iej8f z5m?i1pr%OS-^>gZIr_yVl{u-!pg0ZiW@Hj!hNK0MQQO^LcO3xg7h_^z;D(t4q#GDb z|1mSf7p3MD>+69?G>y~sS%Geno9@lZA_ulE2gFyI-oeVE33lVshtd)9z?7B1!oVPm SVz9vU3#=?^YzfRj83q8S$L!ev diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/corrective-action-plan-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/corrective-action-plan-workbook-217653.xlsx index 013541ed85403a284a7f99c7a749af2706d7aff6..e7569e0d40a2cbf22c2f908e7d59f47fdf0f0508 100644 GIT binary patch delta 668 zcmeC^5oqrb;0^F*W)WdvVBla7;_I2nyS5%kg)iNj8TJ(@c$$%cK@do%Q&_C^iDkJci4cZEk5*sJ0{_!D-qM^E)m1e!r=P!NH|dq>rd>u+uNt~yRxI!SQ}{+}hgR6~oA+5wR+~6p zy7+k7+^N?;z1vrrP;B655%!wD^c`ON?9Tr0s1syTqkMh)rafx& z7)x3A{|l>ULufR}N+7Tm}-^uK$}^ zo*BfN9{7*>8Cb~bKQj{(Sje7{mvnM$zdyEG#@=L2DKkX&^tGg+(1~UL%A*7s!tSS{eRqkA&=ZCI*Hxd<+b{z^H_Q z21do-%nTJd`o$%cIjO~a`k_UlX!XJwI71Zi3wSERQEsHhjHNeV@iAtTFle?}H*el&Zg9{?F4JN*tTiyB)l IGYe1?0KDkarT_o{ delta 653 zcmZqg5$Na<;0^F*W)WdvVBla#jcc38yS5%kg>#kMkNyS}Jk7|!APA&W@{Eh9X>(#3@;{M9j z&$;Bj)%}E|_CY1h#U2M$eC{Usoby^NzV!E(w>kz=8(h0(SszHYYkR-Q{joXGPppXN ztP}6y3A{(o9PPE(V{gCr-09`M;vS_^!jY1?TijjuPQToJCo@NThi2IGoA+5wR$Dk; zIQh_Or%L|Ov}k)Zqx8y&MbpIgJS$g{`}VCwHOq`=teQJ0*!K+F3ZXy1+f;y*PR~5 z$|9!-GJ17fk=_=ddA&g6rC{a)=>|q4MwaP*j4aaAH?TsTydR`jZu)Ij7B#j!W)`3- E0Flq(W&i*H diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-audit-findings-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-audit-findings-workbook-217653.xlsx index 76eca63821940714b32418df5d7938d4e34d1340..57045efadaca9b3f0abb052ed9dec30b94bbe855 100644 GIT binary patch delta 571 zcmX?j+vVVG7v2DGW)=|!1_llW6~3N{yld-$R5;_7$8Wv@1y3_FFbD$al>Fp?qWpql z{p9?jRK1GaoZg8C{SF)Ow8f{M+FN?bPuL|^U%^j7F_80!uGgNmi#!(}S#kIFlB`Fc>xEAvKk zefTfuw}M>f%o8R~30#uElvby>N%+zW)|d9q=FB~8Z!Roa%%LP+@KK0O`hU)hsYfl3 zl@;3E-IdF(|18QxV?$LeGdbr?T_l)o*;+9_!v2@0zQgU=W!L{TM=Z(s;kGVcb^lJK zd5op3`~QX2Gol36=7)@SY|OymYF6UduEfE}?#ryo*VC>O$_T_vK+FupEI`Z(#B4y! z4#XTl%n8I?K+FxqJV49~#C+RzLiwlf0Gn`(S77@wUO`?F5O4c6ML|_&5NrBB6~Sj< zq35cCyP3d3OEm;Pf`w*k3WhL(95elcmY_8UEE1<1YYUo#g>$t9?ZAw6+JeqtylDM{syld-$RCtqJd-7MH;Aut%20w8C)?ovlw)mw#=4trtlJR-v?d~tjBX;A)*&A8XV#kkN^Z516dz0DQ z8+^?1YyMwWmkhtf{g%m&{kdU4bD`;tZQV}4d94L283n#_uI+m1QIcFya8OHX{X>rW zxR?B;E+;mxVd-Adqr^Jd;A8j1*eP!MUjHR;MJAXebc< zx4!yq_qKbcwE3eC-?r4xobEqMZ_+Chqew~J5(ihk6E9u=*=!NLBf9$ZoA*q!u39)= zxcRs(cJlS4-x2R;oY}m`qhgxKo@d#Hvj6OM)_u{u-*Ro{vkQL?&rpwy{2XLcuyLPx z9P6g0`2TAA8BhXi^Fu~EHs;c}wq_-c?MfVs?7qxpac%87p^QMx1jNih%mT!$K+Fcj z>_E%`#GF9P1;pGy%mc){K+LyYCzOBs4zLNwcm=i};}zr;0r9q9QxsHX2C=69QxSXy z7J9BKxSI(qv{XazBUosrreFvoGcfL^f6x-N=72@wbYpEnbFgr(wxAuzvD4RS3p#`K met|Kpbf6*)I)WBpsogq)W)L+xf>vOLp{}4Sm{F-K=m7wjYvK$5 diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-workbook-217653.xlsx index 5ac773a1c6d4cdf4dc8b358f4d3390e33c318626..831880ca5212e51c696d8c6c65997c5e1a7e52d4 100644 GIT binary patch delta 603 zcmdmbDQ@efINktnW)=|!1_llW3BI0*ylXjtRJfyX^Sg z-(r+<@uF+t{c3H^W&arbR`FLlXnd;S74yvU6jDogC6-j=mbF#k?(+QfhOh?Rs5cr6 zrzK+cw%lv8mrhG9>^oSmvo29$Q{(x4I|Q4zFF!uL^5>P3#+2m-E4A0Qe9~C?{cFiY z>)u>e^P{h9e(!Fm5>@B0vb>`r{&cE+&g|GA@ydh2)0WRMT_a-1_0Mi&-Iw^rZP(NC z|2gU;nf#2JQod-f{ zvov2%`?gRJ__AEw0<2hng}51*k+DMD3d~r#LfjS1c)LQ}0|1Kc>NEfV delta 603 zcmdmbDQ@efINktnW)=|!1_lm>oVd1$ylXjtRJiqn!Xpz8Y1NnQlJR-v?d~tjBX;A) z*&A8XV(Q1PdHniTyvgkC4L;`hHUBTGONQU#yjxxKZLhOGZ`;+3+j<;t%}#cn;A~jn zpAmXO>Eh837phll#QhbVb4mVH$AQm#Sh|<=D6vj9DD5`fsnHuZ<=y4&n-63h5YsIa zX-MVC(^8ftUk`If0l9h`E872Z(urm=B2gfmi^D1%X%y zh=qYzWc#*I(RB#PKK;g0acg*#PM2OLZVna>T_$b^W=vZq?hNwO w^gAF1$Xuo6P(}ufu@1)gvRvE(tVVx@xEYv{u|nJm%vicY+!f4tyF%Op08wH85&!@I diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/notes-to-sefa-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/notes-to-sefa-workbook-217653.xlsx index ec328e3e885b972dfa4a2d2ce5029ea730f44d2f..67700c3cd5dcb6afbed125fdc8818ed9fd5b0219 100644 GIT binary patch delta 501 zcmcaNRp8!K0p0*_W)=|!1_llWb-td7yld-$RQL+EyeHp)f~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBnS%(dH+Txc!nRmi((;=T%r@j28dBkqqIC~>YT1@@eHIHB4iZ_|P zy}`#EzvlmCb;Z%CziOCZyp64UNskiiWP{RfLrd-6xGC>0Z{K_%pEPbTdV9hcV`IvRHscZg8@gLDX=uSb-USTr93&#xyP#4*;97!C(LY delta 501 zcmcaNRp8!K0p0*_W)=|!1_lm>inz9kyld-$RCv<087AL=f~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBneuoWs+Tv4R?Jd2OC)%|u&S8#&(;}gc=qVM^Th)4n7MJDwM{UdF z;FEd$|8siu$|=IDzW;bs-99ghXZ6JrGmfX*PBxz4Y*@gTk#a&QF?q-9y%R;|*YW$+ zuUdRZ`AN^zgb7QgsU!+5%QI9pZmosB>tBmvR zfWrnU&W6Q>%6#w7`>+4}DKnedbG5-r?X@kRG**89S~AhPH=qsDwyBn%R)j6yz z@2E|GvUW#(hTLSU^va1v)5P{X%syz|w6Xu>*Tq`@wM_QrmA|#`sXTS6ZFE@^op59g9$7o%Es~%EX2sp;?D>&Zh9~Wi!}!<9;Q#@U@->^U*KS| r12Y6US)9Ro!(oiMoGcb#ksF*WW?+UO7mF2`;m5_|3T8~>V(|a~3>3>O diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/secondary-auditors-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/secondary-auditors-workbook-217653.xlsx index 7ca9f6aad240e10312cce749ad9da0a148fcc885..ca3dd4fce6e8d7a3adf7f6749b7eb7b23cb4d05c 100644 GIT binary patch delta 524 zcmZo{6l!b~;tlX-W)WdvVBlcT;_I2nyS5%kg$sSZRs9Vpc$$%cK@do%Q&_C^iDjOb=ZKXEq>{fc_+#?9rBrV+RI;>N9@Lpvp2G&#ng{o^Z50xc$3-N z8+^?1YyMwWmz;jf^KNy`x4q8(ylq!AZtHoxH9I-+glEHo{*2HQN*9k7+^=4(5%-s| ze$FNNR}Bgh(ao+wZWEh4XY4vUW6o4JeXs8?Z$~DWBy@{ea~&||3!ir3_J{Ptb9A0a z%xaZtQFPGCw|G%hkQ5Mod2h{Ao3{Z zBtJg$ld6?x<+pv+Tgf6^c`9<&=UL*D?t6yq@n5GU|2KT$Ti?z3{}M$$UdhqzuGYNK z`%!6y*N?CHSH)36Yx5&UJ2qxucr`0?Y**%B{OiN4&DYbe9Lfm9OhC-MT{)EHgA_<~ zyI29s6&5h-Z3&AkGl(^vw}RyvSm;{?OAZrQD7Bj9BUmW1h9!g%Wcc(AwJg>guvnS? zxt7HoEUa6{Vh3jA)v-8(^=^SNe!&Q&_C^iDkJci4cZEk53^QLN*mZVxVFo>3ao6<_LhkGvpLZ-tf=R# z7w_Q-d_NEQN^@3JSJgyL^Pi+4tVV?YV5&5s!E*qDLg)vV01U73UNuMcxgTwA+xC?gOv0WtG-Gu|IZR-o)M}QGV4=twmJmje;nO$NvRHG#VrBa0 zS{8G#ux=fT9hi|<$Knjuy9LJh1!FkWvsi$os_R+IAZqGatiX(q^(?MnhGheb2LL|i B*k=F$ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx index 8b3701c670746dc299f8dc8a0a692f8464a0edf9..204ab1fa7dc2cffd8484c7ab5d29218d7ecb28cb 100644 GIT binary patch delta 689 zcmaEGkN4p{Ufuw2W)=|!1_llW8@`^2yld-$RQQ41W#7I71y3_FFbD$al>Fp?qWpql z{p9?jRK1GaoZg9n-iHkYj_*J9qPp65b0|-2oXC6;7VfTm#htlFm+_wT+?}5vwLNbU zzf)Y%|NDR6IcdFZd1q_)``*IkY=^V9dPWPX!H{q3u+9so@jQ*}F{pI32AKwTo2wmg1c|UhXSYgwG z#fNi?{NjGzt^GUUgpz*6sZUcj&sxj+IH~xv-A?VgFYztg%+Id-*}k!3`8n^6`$T7d zI#nTdf~)TD>_1E>!L|7jqa7PFFu3t9CZc;VPL$#Km+47#>pQUMYRLG8JR?w;qe*1QaSwTK_5njr5^+sBw(t6bVH-~ z^!2&S>R<=l%w?8l<_l$?ECBGK);+7Q#Z5twLJFf%alBiqNgcKZ4}W;w7y7eEGgPk)ogtSJxj)zXL35%NGy V2`uQoDxYqW&#cCFJO^kT0{}p2>!Sbw delta 648 zcmaEOkN3enUfuw2W)=|!1_lm>EpcrVdDjX8sqkI-!F#^}1y3_FFbGaJ$YmC(@11zi z@2~++TYT!Py``6OT)TF~ILvWyS|rpFJ*6Ug>#|;<#bx>a(cAJk_+%dc|C}DJ**kq1 z_YWbf{>H4H)dx*;ryR?l%sg>A!$p}3FK6_+Shwxgv+#Xv{N`YG@s;Y9IUAW|CT*K# z;&bSV{;?-(wxk9AI)C8|Q#ONi$&DbEd4=^mHnki)YA^jPMX^0XKji-RNd2_DI~|9A zf4=;BPfgVJi6z+vflEyr;b8pY1LCwxhcW^&({|}l=E?jZk?k8(m}QwktmzlinV*4$ zc4aW%W&{h($YOp87HY|6_J;^v%VDDr-; zlKcVQj7%cT@OYoDkjE^|xOlov9JZ3qtK^H*=cTInr$E*qFOXdUlWz$Xbnbp{ik ze@5tuMT>hM?fV=S=<}Cx&L#CyhZB;~jjlm%6PpBQ?3y_vW`f&!kMA#Mb0(N1G>b)Z z9WdrI*I&5(;rTYtnRA_2)Tf8v{~j6dvu}rB`}gN}UvH_2+TIwl+Hm61i%!)cS1K*z zo}6bAcFPZT_l&;94zdY%WMZ`^ye}=gY{m8F{JaDEx;mSdCX>DMt>f&6_{}* NkJ%N>5Xoou006KNxTF98 delta 470 zcmca}kN3_!Ufuw2W)=|!1_lm>t#NG=dDn6Psqo-A%l}V2q*dRyTgGRWx4XYAkJybH zXK!Rli>V*G=JD%W@g}pkH~5(2*ZjY%E;;>{=iTa>Z+o5ndE2gL+}87WYj(2pglEG7 z|BTQRN*9lQxKO=XBkr%@oJ;brIwly0HMj=3O>FX`hUf*Bdj!ZB~=oYi) zI$+FaKJCKo59x>J=sb~_Ro^PrrYL>n%+X$pJ+^P-pFO?2jC+YyuV<)~?pAl#z3Z2W z@BDmYT0yvu{^tGM8DWJ@OEw?&E%J>!`fcMq)zEWA6N{#C#eLq|)4uWh$*+t5t8Lkv z$#>Sir}9+SEb&i{C${b|j$;kw-2ZRwFD8^=2%i}0SGdAXJac#}Y9NU#S z7(e@fIPH?5j6lq^T{4t;GCxRU`}!1SS!NJx`uTL`XJDZn8O%2r!9tU>m|ud0>a&^s z8JU6MJN;q~vo$=Rr;Fq=n}db@a+&SGTKaRDoxzN&Fotv diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx index 80f1e829469dcd44afff6681b2726565ff60b38e..e156524fc9dedd78dd58bbdb316529547476b2ea 100644 GIT binary patch delta 714 zcmX@ISm4lN0p0*_W)=|!1_llWZ@!+1yld-$RQUX5m6czBf~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBneuoWs+Tv4R?Jd1@fG_Ym_hml?#X!y@x?X$MF3MbVWX0Xvlc%kf zZepIK|LyYyx%g}u|GEq=q@z9wXQ$tw{MxsCn1Gb%2A@-PhL#kQGQU%E7PK( zKKz%ntbo_CYY7vl1TINndRC{HmcHa7_oe!_JT?j58w-!lQ~J9T87rB!iWbh(`?P%H>Icgr=BC$+Us!XA z#Z$e1_M1g}y4rI$pMEuIs?u{E@jVaAjaY@>3%_>P_%E6j86}$CIkvlVFrM;Z2FkXJ zgfap#({_WP#>R9@ zeimy^c(hEfWK*fVS<6jesBXMmLdDjX8sqh>@Bc5+S!PATk41&`q^0SE4_f9;R zb=ZKXEq>{fc_;j~8Th>NKJ6#XBX;A)*&A8XQtHR9dHniTyvgkC4L;`hHUBTGONQU# zyjxxKaj&yKZ`;+3+j<;tubpf>!P&5YKO^*n(#4|%*Q-}+#Ql|F+OZbqu67xOU63K9H<$*Ys- zM(LFki>8U~c~-6@7gpo5PX2HB!neMgv;QTY2@boqN96m(9|g&KI8s>m|MRV5gar>P zID9rgV6LO*#}UV??5^0N3df{dGP&d*}a2@m<{)%+}G@*wWam^sP-$Zu%*@lEYi&Ep^TFa zIYg%C3a|)*MH_*ljCZHc6=0D8^Y;MxR{uF!7~+dkbBgu#KxBY7Ba;X-B#i)l6kcj? z)^Zb=Fl<>E82FK`V|+1PP>@AV9;9e>T#?=upkyx-1A`QbqIc8%1X(n}wp9rNZND{r OnjniBn-)LNI0gXaz2_AG diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx index b5b4e51928c43d8e1be2ad769c720aeb8a7e1c9e..fe708cf0418910a4534e07a1e926b10eb0be1c22 100644 GIT binary patch delta 673 zcmZ2CRbcH@0p0*_W)=|!1_llWN4}nkyld-$RQTDs9)Vwhf~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBneuoWs+Tv4R?Jd1@V8Wtx;sWXuCahq2B&K>VY^~?wBP;ISo;+=> zbQAL={crz&Z`KXJ<$agEhI8+62i;>=Gpxmzew&@+|3uZ{O7#{kv6Bxc?ReW9I_1&^ z=KA$t9N!9f9lMq=aZ2El1g2+oifQUgE^=S0Z_8to;JvZX)m^aS;KN6%JZArH8fbSH z_q=^1)wWRL$Qy6z!ixI3s=3q8M~iz}_D04^>XtgV)~;CI{ipDa)()+(rb5!9o|=So|45#!Xk|V6oWV;{!i`#Q*)*RK^SOg z6bNOUt{2KEIX#Y(g$L|}Tuz|g+69?WJiF!_Yh>zrRje;S>(V5%W(nu*QZ-?v1o!Fwe+EMggh`wC9p6s2%{K$ Xn2lxnLpBzX=`*-k1lWW*fC?A@Q8eXE delta 679 zcmZ2GRbbsz0p0*_W)=|!1_lm>?Qv}rdDqqhsqnZDic#Nyf~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBnS%(dH+Txc!nRmi(7mv@Y(_a44JYqL)oV}4HEvA0#n#Zqi#hc9D z-r!@7U-SR6x@7n*&b!q$-}XBD^R`{hxUI+W*6ifO6PyhT`ZGdLC|x{SaKC!BM%-V< z`Z<^6Uo}iH4r_7^a+}!XIb+w+8FQw(>3e;Dc{?(}B%xc(n(KfuU-+~Ow?CvGo}=?b zVpgkEo1%2r%x>wzit4JGnbZAe>rHrNqPEE>Dx<%vN`JZd&dWE#3WC?1+q|DUBdoA# z!Q{ifMV@ip?YVosQ_g)-da9#aarU~BT-krSopoAY;#;?wpWXMf-OxWW@^jD~(WKh! zJsc^l`~UgYF`|Ul=7)@SY|K02+M1O(wkvTk{_p{D+GRr-ftYE#Y$)@VRUnb=SN=20 zGlN*uzcH~q0}I_}W?^Ci3q4_Fc?lM}z{cXw$P5hr=}H_d)|~Kon4ZSLVg?qU&%q+i zbd_=P2S!o;0B=Sn5oSotflPh?k`v>YY{(&{izXLt)OyRNaTOE8ZdDZq31D=>KtrQo zDC2b9P)5n=v79VCVCUp;vPd(YnBKw3B4YzIH+;MM>#hSp4Ps0T4BRk{K)QkP6%z|X zd{JsnvA!ONM6-1I1CT+Nr~lz(kpmkn%LU}$m~P3%qA3qDc7N5JoZh Q$n-`o7Bw~z4xkJJ0H- diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx index 731629b439860a5a70a6d220435e8eb106472d9f..ef8bd64e03089255e401ae42e919dfa9f76a87c6 100644 GIT binary patch delta 796 zcmdmS%Vqm57v2DGW)=|!1_llWf4-iHylVx4RQQ5tmO`I_f~Of77zC#u)Dje_@11zi z@2~++TYT!Ny``59@daMzzU-%<7|3}<*K5z(MVX6^thjr7^0c+mP0W+@zy1HcSvUNa z_gf}A_UDEH-G!#N*7XPd=Cu~8V9wO5*heUxnDD|L4q@derh*St0M?3H(prcuR9u zR9Dr^oPPe6-K1Bhn|2vRy=v%+S+TtPPv#q~9hza!Z{BA$S#9EY>Ez?K*eTaPCEk7$ z5&L1$8J6>27dxyiTPx;A*#BDZ@Y-j0{(ncEAd?#1_d9j+JH;Jh53b*TH6AH!nD}}& zKVYh&Am#vKP9Ww2Vs0Sj z0b*Vt=G(3o$}h45Y=SMXz;;_+L0%CMZ@aIepei$nHN8|t@EKSrPE~L>6Ie(_L+~S5 zNI+9Cgc0PJ>FHX6)|~K2o4#60&`ci0of$JnS%8g!VU9WjgD@~IVW5HW(ll*BC9oRP z$BtU(eHaaSs?RF` z7JUp9jRHD3{H}PB>0c%WhG>2U23}+X8SNDX87gx0i%TkVQj7H}a&rQ_8JR?wAxRIU zb-VlPt^+_VVnFTOC|c8}`{@WuLENGvD9so!y6LD=5dDjX8sqo0=Z82Yff~Of77zC#u&=M4>@11zi z>xh9sTYT!ty>pjcm2)x|KRUT7!SSQmq?>n>F1MsCxx3wb?e4pxwj4WC{^jqRIl--Z zVeEnD4{X?88RV(m<`2x+pUSSJ&#=gDL8zhskqL+9a)$-(YO}3aYJWv;p-z{S&LpEQ zku!(RtamF5lG>~FH~ocpaf=^Op575r+~U&*$~tG%5H^SeDH3nl)m2xknzp zefcpp((ZMeN7jWCDal(dFpBO}crO0)@fGue&>X+b`?)jBGMg4GKCWx6zW(an+TRCK zBBz8N>t6Fncfa80hwnGPcK`31G52NJUHhKYuq(RjkLDU(3qQ8m!R*89`%lCXfx`?8 zmd*DW?btw!W(AJz3LK0~zRV}%+S*k^8G)Dyh?#+y1&CRJm<@>8ftUk`If0l9h`E87 z2Z(urm~XpkD8I-K=2JiuY+wcnVihy|AeG~;%nL(`SB`Si?z(TRAg1ec(Led(7 zAHhQWnt~yWAjeEk(-O4igvZ+SRa$~(@*wWam^sP! z0xL3o?5K6#hmm3J2LT2Nm&?qw9L0eEAA#Uk~INktnW)=|!1_llWcfOv9yld-$RQTbIYqxv_3Z7CK1c2VY{BP;ISo;+=> zbQAL={crz&Z`KXJ<^7h)j{Uh|KzE_(t#$oDzjdvJDmewd3a;&{Ec)VkXLo|=QWKd2 z_1b?I@^P>}m)_uUN+ZaC(RjbWP07qJO<(3OoXzOQT$ZtMIfs&X!B-(R>Hj%1rXICC zR#wP+cmn^^H{Q~m71dQWGpC=wWjE=S>84#qQLh@hVpc5g{*(DeYlmjo^PBfsO;(#Y zUOM@BTI`hTMdkNxHtc%fI+JtW)J1~jMy`ST{MX29{taJHy6o=u{|aK7Qu`)m>mB{o zR>62h^v|#8zpN;MwfPaF9UC(+xSEwYwkvZmcKR}V@%6N?4`l>mCLm@8Viq7~1!6WJ zW(Q&pAm#*OE+FOxVjdvo1!6uR<_BT{AQl8-As`k8Vv+6ZLq)GW1DU+N!$ypk4Q$5@ zC$a4_oW!4mfW_;li+^MSvx;YlhcSa#)6dKmhiKY5Pkb{(D0`9kU$Cm|#p2P7Acs#s zxkTKW6CUByd6$ZtDS)^$W9BFeurV;qQD6RHta)Ag5$tAq7ME)$n#>T;THuuNQZdiOG@iSvP?%!Sj%!6q7*f>LIH zHzSh>GbBZWOb^=iEJlx+fnkXd0|PIz0~u4NGcFgG0h_L{TwI!Q-*lVh;!+UXfP9DL z^TZh{a`emdi?WmQ^RuxT&gA5J=LFDDXBGwqDHOwF7l}`=UL-C({rqxq9195E=dDqqhsc^3f_mkg%f~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBnS%(dH+Txc!nRmi(myFLVZ+CxL9Z+o5ndE2gL+}7iGYj(2p1ZTqn|BTQRN*9lQxKO=XBkr$Y z{hUkkuR10ehc&qdxlL^HoU!ZZjF>5I`d;5(-i}N#N$3`{<~m@^XFl!1?GNdP=jc3< znAIxPrYPMtvs=2bqWaeUGpCpPihGnw2}ermZgF?rJN1#*sjFE*y+oBD6Xx2T___EGXXI(5VHU=D-g2* zF*^`*05K;Ja{)0o5c2>rFA(zqF+UIs0I?tt3jwh(5Q}VI7b<$~8OY@A?KWb(Y+yU4 zJBe+d?j-&s1T0=RUHl^xm{l}OJd7E{ntpn&I7HKydE%QPLRpK%|AJLzEf$Yv1UY>A zi6!FJobVW*&a+h9OdiCY88b&&fQ^A+jyeN_FfakYKm+5|>4{6ll@vgV-m9r^3iD-T zaQh^{AOTYWq#GK=I3^o%NKN0nRGbIwm@`YorI{CnGEO&GCN47l-%@cwd7vA^H+rR> zzs1VH&|$^*qWeS&tmkL85oubF);8VyOS|(+H!FjuvYoy;?j)!r&})6AuFeKZ>3e^TZh{a`emdi?WmQ^Rq#@A;6oFNrV}ai$I#0oLujm0BUk(VPKF# e(Hu8@<8pCLu#I;?HpWf=xm;Y0ZPyZUpfUjcm`oG^ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx index 573cb2b8828df8b2b545f61a8b6ebd87fe508e5f..65aaee03d988f726fe461140824b594911baae5f 100644 GIT binary patch delta 501 zcmdlmSzyCt0p0*_W)=|!1_lm>Aikc7yld-$RQQYR)pNfA1y3_FFbD$al>Fp?qWpql z{p9?jRK1GaoZg9n-iHkYj_*J9qPp7iv0BSI{Vsp5Cf1|dC*0YVG`a1`lDpf@*G8LP zWLMPR`Tu=gx#HAsLEjnYJHIy!=q@z9blTLjb zV^<_fbUB`hxz%y__T|T?_tf0m<#8(W%&SuwE19)kIos|%A%A>b)9R$sS$|_4HhT%H z$lB&@F8-wT?%nM6HF}YnMdykRO4lj>j57LWx3f;`OMJ^V^SA5%wr})Vo+!ODE$B|1 zKRc&Yx5&UJ2qxucr`0?Y**%BJm$|F%-7TYCY%w7nYO728cblJ=PWEQ!9tf=S^OD6#!XjcXR+ph#l!Rrb{2E6@FI2= sJ22xhJBu?|uO4@c;k- delta 501 zcmdlmSzyCt0p0*_W)=|!1_lm>({XJRdDqqhsqh0vVI1Fpf~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBndCdkqZShMV@%ee2R$LSl_H~Qr;KQ|0$E;PNdtvl#6-LpW0lwltKVO(u5U!)Yc|UhX*kcyY z^#0pVX6@-}kKOCPYR*%or#hm0o@JjjpS1D&$*+sG{;O>{XVU&m{+Law>sjudk*8zs zNUm>M)e`?dtdXX4u0--I&)G1K-p;mrG&fkd{i z`^T)r3}Q{c%*gT#EVPG-MS}?}^qhs|C0OV(D~mrP$hhgM>@3zCuy~lB!Omh17GA{8 tVh3hCW@m8*>(%6dGV(cCEWjeGI9SZUjK>@-R$zt#CyOhXk-^F00RRIL)p`H` diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx index 8513ec3db4611484c4a8b07ca60f6f5cc658f9e5..81e448a9985f41bcac0e8c965ea1cd7bc7fe3cba 100644 GIT binary patch delta 524 zcmeygCLMj$xjX_$}cF^ zPtGq&)vL(O>795m>#zY&Tl~@|^G^6(lJj|W+RI;>N9@Lpvp2G&#ng{o^Z50xc$3-N z8+^?1YyMwWmkhtfdAGXe+g@jX-nOe5xAi#QZadj|g0o?Pe@5sDrHe;DT&P~H5%*WH ze$FNNR~-|Kw=s4v=}}^xY*5;5xKpDyZpypM+czJ`I3T85Cen~Dlc%$A`@{2XmXRk6 zO}I^YoXtdx#mygm`1a+8k^X$MScR*ZJ}IeNF35>~jq}qldcJY(gLM(HY4ze4);wZy zOmDyaWY!)r{@qpcPt1AhdMYyN!>s8|>sht$t6$gZ|2O->w`GD+vP%8K1hK? zw+j}qTwwvTUX-xNGJ{ytIVxD5frUO+u;eg-g%YY+K7xfpYFI)TL55FXUCUz40gILC z?`m1h!NMAKEOuZw8?-(druw)oUjdrL2s*}BB)EBGlW267(J_1d#`QRbo}EAHODJZ-IX z6Z7#O|Nq$>$+O`t(@^}+AM{a zPYRFgoy@K)zi+d_>T%>ej(JZO3ATzvpX~mA^Xp=*|5{tlZTbC8{+Q0gRAsN7x+&-K zd&C`M53a9Yy`KRkv^GCtv}0oihF7yP$9827#veY+7vkF5nY=v{dDjX8sqp(9uerYg1y3_FFbGaJ$YmC(@11xs z>#zY&Tl~@|^ECP{^ZCp=eaugqN9@Lpvp2G&rH&uF=JD%W@g}pkH~5(2*ZjY%E*XA` zGu3uq#p?^f-HE2N*6};-%q>zVVp3SaEG4)$b6NftM4~Yex>o9!*yQ%0R>N8 z&qFFccawb1d94*&`uodWoeNwWoSSV~6C~@KwY^{X{@9%8CsxGs)`|D<1m2owkKB0f z&7U9l{ORRw{3_M0p0Sy#Q`|kTUBBFYC-aTg4$ZLVH}A7vS#9BX;p9WJohtc%{w=KZ zd8MRZaq82Q&23Yg3ODZiJtM~c*LtVq(%sqr6LlVCe(Kfm-k4^7tb4(_2iNysjfVvf zD>!^MKVr0FW1hv^)2z&~U73UNs}G3NE*;7U#7x_zLzyS@gG9D(OktK~2C=4JOlN)u z7TT4;e47z0G$V`oC0M8>o7ta{85rQxujMdXbHGA=x@0c1IaoM2m)Q=iWlAoyGnjD; g#!$#(wg8Jn=P{c>)Z{T+ff*O`m|ejP$$VxH0F9E%4FCWD delta 487 zcmaEOkN4p{Ufuw2W)=|!1_lm>oY=OBylVx4RQUFO$qCCFLby=^_;<9}I=xuo%d@_&!e@>6q?47=h z`-hNKe`8k9>Vu}aQ;y|NW}Y~m;iAlimos`@tlM_$S@=FSeseIp_)0a)yyvVID!Hba zGg?;0_neH(JiF-2{Dm`FuQ8at*szpCC%L{Lm9x3K-t5dtC%%nwEB3wL9Czl<4#DQ{ z&zE1X{JAAh`Q^2QMPAuUI9IIX&f0Zie%rE^*-4f2{>D1&mQweys=T8%{fX}#`;h+5 zN0#E6r-Kh_ZJXwE;_ThyPkvpj7u|L4+U{TSZu6eG9_898HYq=^N4!z%;r0D1^ufmkwo~%nuUTzA=SamKnsFeleZ- z8CYmn2J>x3u+WSw=9gfhmTYE!Mv!sSujMdXbHGA=x@0c1IaoM2m)Q=?n3Buv4Ay%K h#!$#(wg8Jn=P{ds8B_9@t-y?ndCab0hGago2LJ?D%bWlJ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx index 72d2d3e7b3e63b946a3982f940592d59f9b93358..a62b930a311eb821afc96fac1359621289fb4d53 100644 GIT binary patch delta 630 zcmcb6kN3_!Ufuw2W)=|!1_lm>*}OdydDn6Rsqjfnul9VMcv!3clAO=0(_a44JYqL) zoV}4HEvA0#n#Zqi#hc9D-r!@7UGo3BI%nuD&b!q$-}XBD^R`{hxUI+WcH7C$6PyhT z{4+vNELzX=}>jj?-4j}oh}K`FQ4PKDmM3GXg%H$ISYKuotx zq#<1CJ?0YT#{D)h@cq-0 ziPrtOtmQ{v8U5bfkS40mVP$G~cKLy7A75%$0G9d6Hl zcHvL+jN>bAx=mRcvHpEWc*B|p*Y7_OM+t@SnK5&e1=tuE=4^h$Xu`$}43TDKj_t}E zj9+}1=kWHlONBB5G1GRbQ0B?}Ad&4GQkZ3#L9FQ)(wU!ug?45z-(Um_P03<@2^MO| zX7*4zo2Pq^FDJGMjUiGAK=Z%B*F}jo9PpPa*U$W*W@y* z%L8o+|9*WkQvyE|gSemsg9I=JVW6RrXZk0gBCy>uc|c8zrd#AO%h-T4ZFhg&bpWVR zjERAP8>RzDH!vPbXJ&{mO3f+O*8`ErwuAgN0c6nX>1*4Mh1oi76t}k6oZSV8|E{su^q}`1_}cJvs2^^ delta 657 zcmca}kN569Ufuw2W)=|!1_lm>yx6vhylXjuRQNZ^1g>ur4{OzXC8&K}n!KEsjXUQ} zTFzG9ZsFu?mD=Byk+Zjzv^VNs`~PxskM`S^ceeX}-CMYv?QqtXT=AB&*-spvusB>` z-=dW=@kP>(*Ly=n*4Od-z0}|8enL|FppxcdkAo^ccawblycUZu{r&B&j)Bw$*KS$X z2a@gF-Y;^0Y)m|ud0>a&^s z8JU3rJpE!0vo#{5r;Fw?n}Nmsa+#%>t};&kz$iMsA(xp4tiK;5D8@0_kV8s4z?+dt zgc%ZHKRlGeDdDqqhsqonk^mcy*3Z7x;Hwau^#7b0Q;%96 zD=XwZJc0k|8*gdOit4JGnbXhTvYYhEbki=Qs8%P`OW*RCaX;x zFP(fmEq2QFqVoGT8+JW#oyj?G>LS7HnN8E*3%_>P{4bg@H}m^j`wrKxw)`i*{KWpW zRWM!={qrmOFDpu5ZGOmT$Hor|t~ZVM820!uGA#NaFkK;(QKH$6W4jv%<7ppeplrKH zC?gOvZ5Ih;_Fc)mnzv_r%0Fg#W)N$78zajzuuwh|3o{c~sFa1}C0HntmBpVCWWw~# zY%JCsuy~pNjg7?|ENsBeVh3gvva>jY8QVb&khy=^SuDT|3l0`DFr$!z#R|+=$HC$X KW_;sd@c;nmSIz$b delta 517 zcmdltNnqb30p0*_W)=|!1_lm>_Sm+Gyld-$RCwxtzq?<7f~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBnS%(dH+Txc!nRmi(myFLWZ+CxL9=hf<(Z+o5nb=$6HSV^}O&3@wYgvB91-NYzm;s!&TFZVpU!s}V; z)vvbOC_U+!nlNF>G?hf5WqF2b;!DH2FV+6a6U$(|vCz?*vEtanV?tHJ|8izbZk2J~ z9dOtn#o4g9P?;~^|M&FDpIf#zE(x7+WonpH<&-PGtEwhi_s6o9AAM!?dv`;Xs5*z0 z^&OS#PqyB$Guyr7Tx8smIZIjde?%QQ-1YtB*TtIu)lBy0@xQh2sXTS6ZF-XIQvUK| zs~4<$aDD&Pc$C1}{E*R(jUN}}{v2E=l zp^QMxv|S{W*>@#KWP8d#W_e~1YkC_a%QLW0J`)Qw6IiH}h2w8?-(druw)oUndrL1(Q(Ux0TtI!ogcU50#8mHvUEC6wbmiS!&(qg< z4<=61|Mvg)W?k>ME$?ji{kXSqIosi^ExFT;qB_x8q4a~ z>-}E(Z*@K)8Qts}y^gX^my)BtwlF%(?&2_+-FI|7(_J{Y|A|vM{ zXNX-l;JF!aX5S9M_V3S^U$6YR`A|k6X4)%3=-{zQxL7 r2WCjKu{eYECc+rY*;p*VB9GWu%)kt3b`~ozBZ8g970g(`&f)<8m~P0@ delta 500 zcmaFSFYuyYfH%OKnMH(wfq{deD7I}P@7j7G6~6n{y93{Vf~Of77zBZIN`7)cQGP+O zesX?Ms$NBIPVdBnS%(dH+Txc!nRlYkn@#QP(&Xj5Y}`3-(sH))b_*wGtJMCsjGVo# zq`gu9+W(i6d%WK+d1t%t*S&?y*$!uI$rWExHv5Ug6BUOG>|3-_Cca48@p^Bl$oe|| zdcT+YTis7cY9CP2TlrGgyVc!w@A_rpJ0IVeRuHP=zqy_}BdoA# z$>PJlMSk~=e%n~*wQ9~&rKdWgd!A)W@>l1(*sjdM_|u2EIJT`_K9muNnYPP^GQVBUToT*1 zo&PVhJTr(j)sW>GSVW4Eg^3Bonl8`G@)9h>&BEf(2r_MYEGvsO2P_(b-XHIHBKpQq&B+9=Dm%j5lOccE2N#ZR%;@SRIu!1p-IWN)w2&)hPFPfQA{ zn587UpKS2Fvok?usmYsz-d`2JbL`PEPnfV|no1(mvpU5z^(A54m+IT<*fLmeEOzu3 zj5zqmNzAEtFVwbGlj5hi_kg80pV1H=DGDYxA;eUMm_V#bhk& z{!{oSdxzGwGi&YjSMS#To?yD?N$9y#dy;gcyESk8KJx2gJ)_j} z-Lc>KmHaPGlis*b@Xxod3N{ndKfj_sv!aAT_{^9&$^vW*44YpunzAti1Eg7rW4jUu zBa1KddfuLP%}_=lW&&bnAZ7t#Rv=~rVs;?r0Afxc<^p1FAm#yLULfY%t{KX2w}W{D z(1a{rf$dqmg1jOi-u5a*K~-iDYx*J;!DnEhZdJkEOkg2T4Z)9KAzMws5Jr$=rccol zw1!91^y^xJ=3rq#Z9zLQBS>4&8RXFEGe8WGx%XfU6&*nfut>a)pc$AkLr2gG%($u} M=n7`Y>I!-Q09y{#MgRZ+ delta 541 zcmaEQ%jM}U7v2DGW)=|!1_lm>zSy>jylXjtRCxZ?#_tmkY1PkZG4OfiecVr)N9@Lp zvp2G&#p)BXRsMeW-emUn1|M_$n(r5tCCyJ2KV`CGe{LAiUT8e)p4+-VW|l&gd;(Vm zdwXuW+|azUJ3+Nk;=Gc$4OC>S|Vrsx;kUd)Tp&y|0RF9E)Y%VXVY~} zkZjiWmhkzrIngt$sON6I7w_Q-yiX7LN*h*GSJgyL^S74vjB4%7*z!ngfr|K%KT7pd zzuEG-&Q^XaXZ)Hr>%hw=g|lNOT`wxXZ>O^8M7L3DWz?o=p*41IYRmq<*w58_`R=~o z?HfHl?@BdYuN8Oau7p0*#RKc>h2v2|A$(@c9AyDE28PYA7){xjfdSI2#Iaq8gOSCT zd17o^yJjdO5HkTWGZ3=?F)I+W0Wmuea{w_X5OV=BHxTmxF)tAFZPyItx7z_WA&XaF zdls)CuLy{@y-HC~l^MjEzDPyz8Ca-WRd6>GSjbaD@FQ5rR#Py95#*TZQ?vxF;n6hx zx|X0hSXfY7&<@N9(iU_E>zx5(+=DSxbObHHQt>*1W)L+xf>vP0RUJWBFhf>X&;tMj CfYJ~E diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx index 6b64e63b72297fcc21d77e9f20d5e50311d770d8..17460f3867f6b4497d63b6c56ecf6c09c97c5bf6 100644 GIT binary patch delta 633 zcmcb2G49^QINktnW)=|!1_lm>rMx{8dDqqhsqkbz{^qYh!PATk41z#9B|kZ!D8Has zKRLfBRj(p9r+4B(zrzMRZSkqE_Lg4mkzJ%WUC~cLF_80!uGgNmi!v7-S#kIFm1)sX zAO6c(R$$epI}NTuZWEh0XZ&uRF?XVyzQ=#bU6KhV3GHIlT?e?>!u1n3d_3P489668 zNBp`0&(Q9V`q|0{%P&MH#AR^GHx z`Vp%__K&aTKUq-%Yx6@!J2vKJygkiI9NU#R7(0ELSMc_q15QpMy-^?qDm%%LcY1 zz*%g2fV23M5U{xU4DpXlV3zi5@i1l(YkJE(@n>M6-1*|0AwtTF#s7kZl$VG{Gcp6? zd3xhgacd4(gik-RRNNda{BNna9mr$T?U#u=gND4##W$9V Xn}HdEE5xn948IlPu3*Nr72+NMgPaGo delta 633 zcmcb2G49^QINktnW)=|!1_lm>hS;`=yld-$RQU1($C|zZ1y3_FFbD$al>Fp?qWpql z{p9?jRK1GaoZg8Cvkn{Zw8bxdGVg?+6QA1I)a2!2Y}`3-(sH(%b_*wGtJMCsjGSGT z)843m?f=WkJ=srNq-^YdA6%&3oM?P=9lz7g+#;nSCWRHkQIeZHz8Ko@-1F)RuV<-O zzuIo2^rR;=Vd9c$Dv4st@=VplmxisrRQoGWFoX5R!lUz;KJ*;!RsCZ2=jH|#ZA;e8 zL60rck~bC0DMj&PaVrC#_0b*7lW&>h& zAm#vKP9Ww2Vs0Sj0b*Vt<^y7WAQk{(K_C_aVqqW_*}g7RwEsECWwt0p0*_W)=|!1_lm>jl4Y*dDqqhsc_%DueW~#3Z7lXPBNIPX^1eB0~n&)ar2*-+hYQuKHRApX z*3Y>l|Egnx@ixZpB|S>4lMPC{4R>nv#!Y#5dHd!A83)94%S0N|W%6_uZhv^b%`)?~$r@q6qn z(o9zwCx2iRoi4z^!UGnR=KuDW^^ovU>b5e^zF^q21cK6p^2Y~v;m>3wiVdeno z21aHk7KZqu)SO~{JrIeeak@Mwi!|e#>DHVqa$xIXKz#A(6`U-Z@*p=ZeJC9v4@_qX WEDQ|7s0L3zz{#S<7R(NmVE_PG?CZY( delta 659 zcmaDmS>VlN0p0*_W)=|!1_lm>DY0!6dDqqhsqhoK|5CK1c2VZyBP;ISzC3NM zbQAOOAOHW{44d-t(ig`0&hHHanhTA0ZR@`DL)TiUl2hQU;M~5KE+xS`-Zx)1u|3SV zr`}BdSHpzDxF+XIVUyb;XZ*N&;+$7@*wX(zZ%E1q@#$;=VGBXYI!TRG#~v{?!- zpA??fJDI(2_j>t=sULOxS^SFvyPdkMPyYP<=GR5ddeL3?%zi(UKc@3ARo5$0WYZ4Q zIF_kR@&ChW8Bqdj^CLz(HfCUOH7j#$SLR^s^=Ft+HeC)9+5Y+; zvl26iHC=#-} zrPG^$iokZS0cpBB{R9V#j15TBcK6p^2Y@QYm>3wiVLE_x10xd?3qyQSYEH4f9*9J? z9po%{iFfAppFfa(C8a(|VCyN?e5IazY F0RWP7=)eE~ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx index 9807df741217bd47f36717bc4cb3c642ff7022a2..361dd03a2500662ac3cf561b3134ae0183cdc610 100644 GIT binary patch delta 524 zcmZ3~B($hWh&RBSnMH(wfq{cz3vbUv-nI2WD!eCD>)w8C)?ovlw)mw_=AH1nBV*G=JD%W@g}pk zH~5(2*ZjY%E*XA{^KNy`x4q8(ylq!AZtHQp-FC9`1ZTqn|BTQRN*9lQxKO=XBkr$Y z{hUkkuR10eZ)5CU(xb#W*`T!BaHmFZ+?02hw{Jd>aX?JBOr#-QCQoPK_J`-&EF(`C znsA%)IGc$Wi<>|C@a@YFBmMbiu?kl+eNs}lT#ys}8t129^nBym2kRnY)9S@9ta-%Z znBIQ-$*etM{JX2>pP2L1^;Bfkhgs8`*0XBgSHG^+|8Mq%Z_6xA{u?~|cqK>q)K-n3 zb3bZL5dHJZ{3k0)Xl;JPXvfA346kNoj_t}Ej6Z#txAOM1%ZD-oF%u9oZu}E|F8j1+y7Hns$VbBb?w>}=P<{?X^~Jz^puKd*Ut-&thl>>xo)&M z3wz(k-#<34y>g2CDU%)hbHjl4LgQKY+!p>ZvlOUg6u8RR+jG<9hUT4}2`awlH?Zs1 zUv0mm{G=x|VZxGWDv4st>P*$tmxisrRNq!7n8A8uv7@(O#j%HngxG}t=R{1JDwDiD z=&(VG^TvY9ZZh}g$;D55dik8&q_D(GSE4vY1152wt(j=wSIZlI^q0x+-3?X7z5*p* zcKDt3iRB4dt z_U#2MS6INTzET!hW)N%ox(b$OV4>-iEICYIA+{Qpk6@u+H7p^F%)rQ)9$3d>%>j#* z=~L=h%)!Fv>R9YRmQUxeXK@DW4TUjg!x#_iSuDU(G7T(d5H$@fR$#`Y1{PN^<5mNU F2LPQ(*qi_W diff --git a/backend/census_historical_migration/historic_data_loader.py b/backend/census_historical_migration/historic_data_loader.py index 015cb30e05..ae7c12c5f5 100644 --- a/backend/census_historical_migration/historic_data_loader.py +++ b/backend/census_historical_migration/historic_data_loader.py @@ -33,21 +33,23 @@ def load_historic_data_for_year(audit_year, page_size, pages): # Migrate a single submission run_end_to_end(user, submission, result) - result_log[(audit_year, submission.DBKEY)] = result + result_log[(submission.AUDITYEAR, submission.DBKEY)] = result total_count += 1 - migration_status = "SUCCESS" - if len(result["errors"]) > 0: - migration_status = "FAILURE" + has_failed = len(result["errors"]) > 0 - record_migration_status(audit_year, submission.DBKEY, migration_status) + record_migration_status(audit_year, submission.DBKEY, has_failed) - if len(result["errors"]) > 0: + if has_failed: error_count += 1 if total_count % 5 == 0: print(f"Processed = {total_count}, Errors = {error_count}") - if error_count > 5: - break + + print_results(result_log, error_count, total_count) + + +def print_results(result_log, error_count, total_count): + """Prints the results of the migration""" print("********* Loader Summary ***************") @@ -75,7 +77,8 @@ def create_or_get_user(): return user -def record_migration_status(audit_year, dbkey, status): +def record_migration_status(audit_year, dbkey, has_failed): + status = "FAILURE" if has_failed else "SUCCESS" ReportMigrationStatus( audit_year=audit_year, dbkey=dbkey, diff --git a/backend/census_historical_migration/management/commands/historic_data_migrator.py b/backend/census_historical_migration/management/commands/historic_data_migrator.py index e31ba14125..c196a99b00 100644 --- a/backend/census_historical_migration/management/commands/historic_data_migrator.py +++ b/backend/census_historical_migration/management/commands/historic_data_migrator.py @@ -2,13 +2,14 @@ import logging import sys from census_historical_migration.sac_general_lib.utils import ( - normalize_year_string, + normalize_year_string_or_exit, ) from census_historical_migration.workbooklib.excel_creation_utils import ( get_audit_header, ) from census_historical_migration.historic_data_loader import ( create_or_get_user, + print_results, record_migration_status, ) from census_historical_migration.workbooklib.end_to_end_core import run_end_to_end @@ -28,7 +29,9 @@ def initiate_migration(self, dbkeys_str, years_str): dbkeys = dbkeys_str.split(",") if years_str: - years = [normalize_year_string(year) for year in years_str.split(",")] + years = [ + normalize_year_string_or_exit(year) for year in years_str.split(",") + ] if len(dbkeys) != len(years): logger.error( "Received {} dbkeys and {} years. Must be equal. Exiting.".format( @@ -43,6 +46,8 @@ def initiate_migration(self, dbkeys_str, years_str): logger.info( f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}" ) + result_log = {} + total_count = error_count = 0 for dbkey, year in zip(dbkeys, years): logger.info("Running {}-{} end-to-end".format(dbkey, year)) result = {"success": [], "errors": []} @@ -53,13 +58,15 @@ def initiate_migration(self, dbkeys_str, years_str): continue run_end_to_end(user, audit_header, result) - logger.info(result) + result_log[(year, dbkey)] = result + total_count += 1 + has_failed = len(result["errors"]) > 0 + if has_failed: + error_count += 1 - migration_status = "SUCCESS" - if len(result["errors"]) > 0: - migration_status = "FAILURE" + record_migration_status(year, dbkey, has_failed) - record_migration_status(year, dbkey, migration_status) + print_results(result_log, error_count, total_count) def handle(self, *args, **options): dbkeys_str = options["dbkeys"] diff --git a/backend/census_historical_migration/management/commands/historic_workbook_generator.py b/backend/census_historical_migration/management/commands/historic_workbook_generator.py index 603cc8ec5c..53618b4077 100644 --- a/backend/census_historical_migration/management/commands/historic_workbook_generator.py +++ b/backend/census_historical_migration/management/commands/historic_workbook_generator.py @@ -1,7 +1,9 @@ from census_historical_migration.workbooklib.excel_creation_utils import ( get_audit_header, ) -from census_historical_migration.sac_general_lib.utils import normalize_year_string +from census_historical_migration.sac_general_lib.utils import ( + normalize_year_string_or_exit, +) from census_historical_migration.workbooklib.workbook_builder import ( generate_workbook, ) @@ -48,7 +50,7 @@ def handle(self, *args, **options): # noqa: C901 logger.info(e) logger.info(f"Could not create directory {out_basedir}") sys.exit() - year = normalize_year_string(options["year"]) + year = normalize_year_string_or_exit(options["year"]) outdir = os.path.join(out_basedir, f'{options["dbkey"]}-{year[-2:]}') if not os.path.exists(outdir): diff --git a/backend/census_historical_migration/management/commands/run_paginated_migration.py b/backend/census_historical_migration/management/commands/run_paginated_migration.py index 24c069ff78..ca8857c7a4 100644 --- a/backend/census_historical_migration/management/commands/run_paginated_migration.py +++ b/backend/census_historical_migration/management/commands/run_paginated_migration.py @@ -1,4 +1,6 @@ -from census_historical_migration.sac_general_lib.utils import normalize_year_string +from census_historical_migration.sac_general_lib.utils import ( + normalize_year_string_or_exit, +) from ...historic_data_loader import load_historic_data_for_year from django.core.management.base import BaseCommand @@ -27,7 +29,7 @@ def add_arguments(self, parser): parser.add_argument("--pages", type=str, required=False, default="1") def handle(self, *args, **options): - year = normalize_year_string(options.get("year")) + year = normalize_year_string_or_exit(options.get("year")) try: pages_str = options["pages"] diff --git a/backend/census_historical_migration/sac_general_lib/audit_information.py b/backend/census_historical_migration/sac_general_lib/audit_information.py index 96a0c418c6..cab065b92f 100644 --- a/backend/census_historical_migration/sac_general_lib/audit_information.py +++ b/backend/census_historical_migration/sac_general_lib/audit_information.py @@ -1,5 +1,6 @@ import re +from ..transforms.xform_string_to_int import string_to_int from ..transforms.xform_string_to_bool import string_to_bool from ..exception_utils import DataMigrationError from ..transforms.xform_string_to_string import string_to_string @@ -8,18 +9,28 @@ from ..base_field_maps import FormFieldMap, FormFieldInDissem from ..sac_general_lib.utils import ( create_json_from_db_object, + is_single_word, ) import audit.validators from django.conf import settings +def xform_apply_default_thresholds(value): + """Applies default threshold when value is None.""" + str_value = string_to_string(value) + if str_value == "": + # FIXME-MSHD: This is a transformation that we may want to record + return settings.DOLLAR_THRESHOLD + return string_to_int(str_value) + + mappings = [ FormFieldMap( "dollar_threshold", "DOLLARTHRESHOLD", FormFieldInDissem, - settings.DOLLAR_THRESHOLD, - int, + None, + int, # FIXME-MSHD: If team approves, we can change this to xform_apply_default_thresholds ), FormFieldMap( "is_going_concern_included", "GOINGCONCERN", FormFieldInDissem, None, bool @@ -52,7 +63,7 @@ None, bool, ), - FormFieldMap("is_low_risk_auditee", "LOWRISK", FormFieldInDissem, False, bool), + FormFieldMap("is_low_risk_auditee", "LOWRISK", FormFieldInDissem, None, bool), FormFieldMap("agencies", "PYSCHEDULE", "agencies_with_prior_findings", [], list), ] @@ -69,21 +80,24 @@ def _get_agency_prefixes(dbkey, year): def xform_framework_basis(basis): - """Transforms the framework basis from Census format to FAC format.""" - mappings = { - r"cash": "cash_basis", - # FIXME-MSHD: `regulatory` could mean tax_basis or contractual_basis - # or a new basis we don't have yet in FAC validation schema, I don't the answer. - # Defaulting to `contractual_basis` until team decide ????? - r"regulatory": "contractual_basis", - # r"????": "tax_basis", FIXME-MSHD: Could not find any instance of this in historic data - r"other": "other_basis", - } - # Check each pattern in the mappings with case-insensitive search - for pattern, value in mappings.items(): - if re.search(pattern, basis, re.IGNORECASE): - # FIXME-MSHD: This is a transformation that we may want to record - return value + """Transforms the framework basis from Census format to FAC format. + For context, see ticket #2912. + """ + basis = string_to_string(basis) + if is_single_word(basis): + # FIXME-MSHD: Update validation schema to include `regulatory_basis` + mappings = { + r"cash": "cash_basis", + r"contractual": "contractual_basis", + r"regulatory": "regulatory_basis", + r"tax": "tax_basis", + r"other": "other_basis", + } + # Check each pattern in the mappings with case-insensitive search + for pattern, value in mappings.items(): + if re.search(pattern, basis, re.IGNORECASE): + # FIXME-MSHD: This is a transformation that we may want to record + return value raise DataMigrationError( f"Could not find a match for historic framework basis: '{basis}'" @@ -91,7 +105,9 @@ def xform_framework_basis(basis): def xform_census_keys_to_fac_options(census_keys, fac_options): - """Maps the census keys to FAC options.""" + """Maps the census keys to FAC options. + For context, see ticket #2912. + """ if "U" in census_keys: fac_options.append("unmodified_opinion") @@ -103,7 +119,7 @@ def xform_census_keys_to_fac_options(census_keys, fac_options): fac_options.append("disclaimer_of_opinion") -def _get_sp_framework_gaap_results(audit_header): +def xform_build_sp_framework_gaap_results(audit_header): """Returns the SP Framework and GAAP results for a given audit header.""" sp_framework_gaap_data = string_to_string(audit_header.TYPEREPORT_FS).upper() @@ -130,7 +146,7 @@ def _get_sp_framework_gaap_results(audit_header): sp_framework_opinions, sp_framework_gaap_results["sp_framework_opinions"] ) sp_framework_gaap_results["sp_framework_basis"] = [] - basis = xform_framework_basis(string_to_string(audit_header.SP_FRAMEWORK)) + basis = xform_framework_basis(audit_header.SP_FRAMEWORK) sp_framework_gaap_results["sp_framework_basis"].append(basis) return sp_framework_gaap_results @@ -152,8 +168,7 @@ def _xform_agencies(audit_info): def audit_information(audit_header): """Generates audit information JSON.""" - - results = _get_sp_framework_gaap_results(audit_header) + results = xform_build_sp_framework_gaap_results(audit_header) agencies_prefixes = _get_agency_prefixes(audit_header.DBKEY, audit_header.AUDITYEAR) audit_info = create_json_from_db_object(audit_header, mappings) audit_info = { diff --git a/backend/census_historical_migration/sac_general_lib/auditee_certification.py b/backend/census_historical_migration/sac_general_lib/auditee_certification.py index 17d3080aa3..6a2294aa36 100644 --- a/backend/census_historical_migration/sac_general_lib/auditee_certification.py +++ b/backend/census_historical_migration/sac_general_lib/auditee_certification.py @@ -7,17 +7,17 @@ # The following fields represent checkboxes on the auditee certification form. # Since all checkboxes must be checked (meaning all fields are set to True), -# the default value for these fields is set to True. These fields are not disseminated. +# the default value for these fields is set to `Y`. These fields are not disseminated. # They are set to ensure that the record passes validation when saved. auditee_certification_mappings = [ - FormFieldMap("has_no_PII", None, FormFieldInDissem, True, bool), - FormFieldMap("has_no_BII", None, FormFieldInDissem, True, bool), - FormFieldMap("meets_2CFR_specifications", None, FormFieldInDissem, True, bool), - FormFieldMap("is_2CFR_compliant", None, FormFieldInDissem, True, bool), - FormFieldMap("is_complete_and_accurate", None, FormFieldInDissem, True, bool), - FormFieldMap("has_engaged_auditor", None, FormFieldInDissem, True, bool), - FormFieldMap("is_issued_and_signed", None, FormFieldInDissem, True, bool), - FormFieldMap("is_FAC_releasable", None, FormFieldInDissem, True, bool), + FormFieldMap("has_no_PII", None, FormFieldInDissem, "Y", bool), + FormFieldMap("has_no_BII", None, FormFieldInDissem, "Y", bool), + FormFieldMap("meets_2CFR_specifications", None, FormFieldInDissem, "Y", bool), + FormFieldMap("is_2CFR_compliant", None, FormFieldInDissem, "Y", bool), + FormFieldMap("is_complete_and_accurate", None, FormFieldInDissem, "Y", bool), + FormFieldMap("has_engaged_auditor", None, FormFieldInDissem, "Y", bool), + FormFieldMap("is_issued_and_signed", None, FormFieldInDissem, "Y", bool), + FormFieldMap("is_FAC_releasable", None, FormFieldInDissem, "Y", bool), ] # auditee_certification_date_signed is not disseminated; it is set to ensure that the record passes validation when saved. diff --git a/backend/census_historical_migration/sac_general_lib/auditor_certification.py b/backend/census_historical_migration/sac_general_lib/auditor_certification.py index ba73bb7f3e..c6ecd00fd5 100644 --- a/backend/census_historical_migration/sac_general_lib/auditor_certification.py +++ b/backend/census_historical_migration/sac_general_lib/auditor_certification.py @@ -7,14 +7,14 @@ # The following fields represent checkboxes on the auditor certification form. # Since all checkboxes must be checked (meaning all fields are set to True), -# the default value for these fields is set to True. These fields are not disseminated. +# the default value for these fields is set to `Y`. These fields are not disseminated. # They are set to ensure that the record passes validation when saved auditor_certification_mappings = [ - FormFieldMap("is_OMB_limited", None, FormFieldInDissem, True, bool), - FormFieldMap("is_auditee_responsible", None, FormFieldInDissem, True, bool), - FormFieldMap("has_used_auditors_report", None, FormFieldInDissem, True, bool), - FormFieldMap("has_no_auditee_procedures", None, FormFieldInDissem, True, bool), - FormFieldMap("is_FAC_releasable", None, FormFieldInDissem, True, bool), + FormFieldMap("is_OMB_limited", None, FormFieldInDissem, "Y", bool), + FormFieldMap("is_auditee_responsible", None, FormFieldInDissem, "Y", bool), + FormFieldMap("has_used_auditors_report", None, FormFieldInDissem, "Y", bool), + FormFieldMap("has_no_auditee_procedures", None, FormFieldInDissem, "Y", bool), + FormFieldMap("is_FAC_releasable", None, FormFieldInDissem, "Y", bool), ] # auditor_certification_date_signed is not disseminated; it is set to ensure that the record passes validation when saved. diff --git a/backend/census_historical_migration/sac_general_lib/general_information.py b/backend/census_historical_migration/sac_general_lib/general_information.py index 74d9ea9837..b7f55687d3 100644 --- a/backend/census_historical_migration/sac_general_lib/general_information.py +++ b/backend/census_historical_migration/sac_general_lib/general_information.py @@ -1,6 +1,9 @@ +import json import audit.validators from datetime import timedelta +from ..workbooklib.excel_creation_utils import xform_add_hyphen_to_zip + from ..transforms.xform_string_to_string import string_to_string from ..exception_utils import DataMigrationError from ..sac_general_lib.utils import ( @@ -10,6 +13,7 @@ from ..sac_general_lib.utils import ( create_json_from_db_object, ) +from django.conf import settings import re PERIOD_DICT = {"A": "annual", "B": "biennial", "O": "other"} @@ -21,15 +25,16 @@ def xform_entity_type(phrase): - """Transforms the entity type from Census format to FAC format.""" + """Transforms the entity type from Census format to FAC format. + For context, see ticket #2912. + """ mappings = { r"institution\s+of\s+higher\s+education": "higher-ed", r"non-?profit": "non-profit", r"local\s+government": "local", r"state": "state", r"unknown": "unknown", - r"": "none", - # r"": "tribal" FIXME-MSHD: what is being used for tribal in Census table? + r"trib(e|al)": "tribal", } new_phrase = string_to_string(phrase) @@ -38,7 +43,6 @@ def xform_entity_type(phrase): if re.search(pattern, new_phrase, re.IGNORECASE): # FIXME-MSHD: This is a transformation that we may want to record return value - # FIXME-MSHD: We could default to unknown here instead of raising an error ??? team decision raise DataMigrationError( f"Could not find a match for historic entity type '{phrase}'" ) @@ -72,22 +76,24 @@ def xform_entity_type(phrase): FormFieldMap("auditor_country", "CPACOUNTRY", FormFieldInDissem, None, str), FormFieldMap("auditor_ein", "AUDITOR_EIN", FormFieldInDissem, None, str), FormFieldMap( - "auditor_ein_not_an_ssn_attestation", None, None, True, bool + "auditor_ein_not_an_ssn_attestation", None, None, "Y", bool ), # Not in DB, not disseminated, needed for validation FormFieldMap("auditor_email", "CPAEMAIL", FormFieldInDissem, None, str), FormFieldMap("auditor_firm_name", "CPAFIRMNAME", FormFieldInDissem, None, str), FormFieldMap("auditor_phone", "CPAPHONE", FormFieldInDissem, None, str), FormFieldMap("auditor_state", "CPASTATE", FormFieldInDissem, None, str), - FormFieldMap("auditor_zip", "CPAZIPCODE", FormFieldInDissem, None, str), + FormFieldMap( + "auditor_zip", "CPAZIPCODE", FormFieldInDissem, None, xform_add_hyphen_to_zip + ), FormFieldMap("ein", "EIN", "auditee_ein", None, str), FormFieldMap( - "ein_not_an_ssn_attestation", None, None, True, bool + "ein_not_an_ssn_attestation", None, None, "Y", bool ), # Not in DB, not disseminated, needed for validation FormFieldMap( - "is_usa_based", None, None, True, bool + "is_usa_based", None, None, "Y", bool ), # Not in DB, not disseminated, needed for validation FormFieldMap( - "met_spending_threshold", None, None, True, bool + "met_spending_threshold", None, None, "Y", bool ), # Not in DB, not disseminated, needed for validation FormFieldMap( "multiple_eins_covered", "MULTIPLEEINS", None, None, bool @@ -123,23 +129,48 @@ def _census_audit_type(s): return AUDIT_TYPE_DICT[s] -def _xform_country(general_information): +def xform_country(general_information, audit_header): """Transforms the country from Census format to FAC format.""" - general_information["auditor_country"] = ( - "USA" if general_information.get("auditor_country") == "US" else "non-USA" - ) + auditor_country = general_information.get("auditor_country").upper() + if auditor_country in ["US", "USA"]: + general_information["auditor_country"] = "USA" + elif auditor_country == "": + valid_file = open(f"{settings.BASE_DIR}/schemas/source/base/States.json") + valid_json = json.load(valid_file) + auditor_state = string_to_string(audit_header.CPASTATE).upper() + if auditor_state in valid_json["UnitedStatesStateAbbr"]: + general_information["auditor_country"] = "USA" + else: + raise DataMigrationError( + f"Unable to determine auditor country. Invalid state: {auditor_state}" + ) + else: + raise DataMigrationError( + f"Unable to determine auditor country. Unknown code: {auditor_country}" + ) + return general_information -def _xform_auditee_fiscal_period_end(general_information): +def xform_auditee_fiscal_period_end(general_information): """Transforms the fiscal period end from Census format to FAC format.""" - general_information["auditee_fiscal_period_end"] = xform_census_date_to_datetime( - general_information.get("auditee_fiscal_period_end") - ).strftime("%Y-%m-%d") + if general_information.get("auditee_fiscal_period_end"): + general_information[ + "auditee_fiscal_period_end" + ] = xform_census_date_to_datetime( + general_information.get("auditee_fiscal_period_end") + ).strftime( + "%Y-%m-%d" + ) + else: + raise DataMigrationError( + f"Auditee fiscal period end is empty: {general_information.get('auditee_fiscal_period_end')}" + ) + return general_information -def _xform_auditee_fiscal_period_start(general_information): +def xform_auditee_fiscal_period_start(general_information): """Constructs the fiscal period start from the fiscal period end""" fiscal_start_date = xform_census_date_to_datetime( general_information.get("auditee_fiscal_period_end") @@ -150,19 +181,29 @@ def _xform_auditee_fiscal_period_start(general_information): return general_information -def _xform_audit_period_covered(general_information): +def xform_audit_period_covered(general_information): """Transforms the period covered from Census format to FAC format.""" - general_information["audit_period_covered"] = _period_covered( - general_information.get("audit_period_covered") - ) + if general_information.get("audit_period_covered"): + general_information["audit_period_covered"] = _period_covered( + general_information.get("audit_period_covered").upper() + ) + else: + raise DataMigrationError( + f"Audit period covered is empty: {general_information.get('audit_period_covered')}" + ) return general_information -def _xform_audit_type(general_information): +def xform_audit_type(general_information): """Transforms the audit type from Census format to FAC format.""" - general_information["audit_type"] = _census_audit_type( - general_information.get("audit_type") - ) + if general_information.get("audit_type"): + general_information["audit_type"] = _census_audit_type( + general_information.get("audit_type").upper() + ) + else: + raise DataMigrationError( + f"Audit type is empty: {general_information.get('audit_type')}" + ) return general_information @@ -173,16 +214,19 @@ def general_information(audit_header): # List of transformation functions transformations = [ - _xform_auditee_fiscal_period_start, - _xform_auditee_fiscal_period_end, - _xform_country, - _xform_audit_period_covered, - _xform_audit_type, + xform_auditee_fiscal_period_start, + xform_auditee_fiscal_period_end, + xform_country, + xform_audit_period_covered, + xform_audit_type, ] # Apply transformations for transform in transformations: - general_information = transform(general_information) + if transform == xform_country: + general_information = transform(general_information, audit_header) + else: + general_information = transform(general_information) # verify that the created object validates against the schema audit.validators.validate_general_information_complete_json(general_information) diff --git a/backend/census_historical_migration/sac_general_lib/report_id_generator.py b/backend/census_historical_migration/sac_general_lib/report_id_generator.py index 4165f00d9c..f27de719fb 100644 --- a/backend/census_historical_migration/sac_general_lib/report_id_generator.py +++ b/backend/census_historical_migration/sac_general_lib/report_id_generator.py @@ -1,15 +1,20 @@ +from ..transforms.xform_string_to_string import string_to_string from ..sac_general_lib.utils import ( xform_census_date_to_datetime, ) def xform_dbkey_to_report_id(audit_header): + """Constructs the report ID from the DBKEY. + The report ID is constructed as follows: + YYYY-MM-CENSUS-DBKEY.""" # month = audit_header.fyenddate.split('-')[1] # 2022JUN0001000003 # We start new audits at 1 million. # So, we want 10 digits, and zero-pad for # historic DBKEY report_ids - dt = xform_census_date_to_datetime(audit_header.FYENDDATE) - return ( - f"{audit_header.AUDITYEAR}-{dt.month:02}-CENSUS-{audit_header.DBKEY.zfill(10)}" - ) + dbkey = string_to_string(audit_header.DBKEY) + year = string_to_string(audit_header.AUDITYEAR) + fy_end_date = string_to_string(audit_header.FYENDDATE) + dt = xform_census_date_to_datetime(fy_end_date) + return f"{year}-{dt.month:02}-CENSUS-{dbkey.zfill(10)}" diff --git a/backend/census_historical_migration/sac_general_lib/utils.py b/backend/census_historical_migration/sac_general_lib/utils.py index 3b781bcaa0..4bfafeee8f 100644 --- a/backend/census_historical_migration/sac_general_lib/utils.py +++ b/backend/census_historical_migration/sac_general_lib/utils.py @@ -15,7 +15,7 @@ def create_json_from_db_object(gobj, mappings): """Constructs a JSON object from a database object using a list of mappings.""" json_obj = {} for mapping in mappings: - if mapping.in_db is not None: + if mapping.in_db: value = getattr(gobj, mapping.in_db, mapping.default) else: value = mapping.default @@ -40,15 +40,26 @@ def create_json_from_db_object(gobj, mappings): def xform_census_date_to_datetime(date_string): """Convert a census date string from '%m/%d/%Y %H:%M:%S' format to 'YYYY-MM-DD' format.""" # Parse the string into a datetime object - dt = datetime.strptime(date_string, "%m/%d/%Y %H:%M:%S") + try: + # Parse the string into a datetime object + dt = datetime.strptime(date_string, "%m/%d/%Y %H:%M:%S") + except ValueError: + # Raise a custom exception or a ValueError with a descriptive message + raise ValueError( + f"Date string '{date_string}' is not in the expected format '%m/%d/%Y %H:%M:%S'" + ) + # Extract and return the date part return dt.date() -def normalize_year_string(year_string): +def normalize_year_string_or_exit(year_string): """ - Normalizes a year string to a four-digit year format. + Normalizes a year string to a four-digit year format or exits the program if the year string is invalid. """ + # Do not use this function elsewhere to normilize year strings + # or it will quit the program if the year is invalid. + # The goal here is to quickly fail django commands if the year is invalid. try: year = int(year_string) except ValueError: @@ -62,3 +73,8 @@ def normalize_year_string(year_string): else: logger.error("Invalid year string. Audit year must be between 2016 and 2022") sys.exit(-1) + + +def is_single_word(s): + """Returns True if the string is a single word, False otherwise.""" + return len(s.split()) == 1 diff --git a/backend/census_historical_migration/test_additional_eins_xforms.py b/backend/census_historical_migration/test_additional_eins_xforms.py new file mode 100644 index 0000000000..1ab7f82825 --- /dev/null +++ b/backend/census_historical_migration/test_additional_eins_xforms.py @@ -0,0 +1,20 @@ +from django.test import SimpleTestCase + +from .workbooklib.additional_eins import xform_remove_trailing_decimal_zero + + +class TestXformRemoveTrailingDecimalZero(SimpleTestCase): + def test_with_trailing_decimal_zero(self): + self.assertEqual(xform_remove_trailing_decimal_zero("12345.0"), "12345") + self.assertEqual(xform_remove_trailing_decimal_zero("67890.0"), "67890") + + def test_without_trailing_decimal_zero(self): + self.assertEqual(xform_remove_trailing_decimal_zero("12345"), "12345") + with self.assertRaises(ValueError): + xform_remove_trailing_decimal_zero("67890.5") + + def test_empty_string(self): + self.assertEqual(xform_remove_trailing_decimal_zero(""), "") + + def test_non_string_input(self): + self.assertEqual(xform_remove_trailing_decimal_zero(None), "") diff --git a/backend/census_historical_migration/test_audit_information_xforms.py b/backend/census_historical_migration/test_audit_information_xforms.py new file mode 100644 index 0000000000..242ff5bc05 --- /dev/null +++ b/backend/census_historical_migration/test_audit_information_xforms.py @@ -0,0 +1,125 @@ +from django.test import SimpleTestCase + +from .sac_general_lib.audit_information import ( + xform_build_sp_framework_gaap_results, + xform_framework_basis, +) +from .exception_utils import DataMigrationError + + +class TestXformBuildSpFrameworkGaapResults(SimpleTestCase): + class MockAuditHeader: + def __init__( + self, + DBKEY, + TYPEREPORT_FS, + SP_FRAMEWORK_REQUIRED, + TYPEREPORT_SP_FRAMEWORK, + SP_FRAMEWORK, + ): + self.DBKEY = DBKEY + self.TYPEREPORT_FS = TYPEREPORT_FS + self.SP_FRAMEWORK_REQUIRED = SP_FRAMEWORK_REQUIRED + self.TYPEREPORT_SP_FRAMEWORK = TYPEREPORT_SP_FRAMEWORK + self.SP_FRAMEWORK = SP_FRAMEWORK + + def _mock_audit_header(self): + """Returns a mock audit header with all necessary fields.""" + return self.MockAuditHeader( + DBKEY="123456789", + TYPEREPORT_FS="UQADS", + SP_FRAMEWORK_REQUIRED="Y", + TYPEREPORT_SP_FRAMEWORK="UQAD", + SP_FRAMEWORK="cash", + ) + + def test_normal_operation_with_all_opinions(self): + """Test that the function returns the correct results when all opinions are present.""" + audit_header = self._mock_audit_header() + result = xform_build_sp_framework_gaap_results(audit_header) + self.assertIn("unmodified_opinion", result["gaap_results"]) + self.assertIn("qualified_opinion", result["gaap_results"]) + self.assertIn("adverse_opinion", result["gaap_results"]) + self.assertIn("disclaimer_of_opinion", result["gaap_results"]) + self.assertTrue(result["is_sp_framework_required"]) + self.assertIn("cash_basis", result["sp_framework_basis"]) + + def test_missing_some_sp_framework_details(self): + """Test that the function returns the correct results when some purpose framework details are missing.""" + audit_header = self._mock_audit_header() + audit_header.TYPEREPORT_FS = "UQ" + result = xform_build_sp_framework_gaap_results(audit_header) + self.assertIn("unmodified_opinion", result["gaap_results"]) + self.assertIn("qualified_opinion", result["gaap_results"]) + self.assertNotIn("adverse_opinion", result["gaap_results"]) + self.assertNotIn("disclaimer_of_opinion", result["gaap_results"]) + self.assertNotIn("is_sp_framework_required", result) + self.assertNotIn("sp_framework_basis", result) + + def test_missing_all_sp_framework_details(self): + """Test that the function raises an exception when all special purpose framework details are missing.""" + audit_header = self._mock_audit_header() + audit_header.TYPEREPORT_FS = "" + with self.assertRaises(DataMigrationError): + xform_build_sp_framework_gaap_results(audit_header) + + def test_incorrect_framework_basis(self): + """Test that the function raises an exception when the special purpose framework basis is incorrect.""" + audit_header = self._mock_audit_header() + audit_header.SP_FRAMEWORK = "incorrect_basis" + with self.assertRaises(DataMigrationError): + xform_build_sp_framework_gaap_results(audit_header) + + +class TestXformFrameworkBasis(SimpleTestCase): + def test_valid_bases(self): + """Test that the function returns the correct results for each basis in lower cases.""" + self.assertEqual(xform_framework_basis("cash"), "cash_basis") + self.assertEqual(xform_framework_basis("contractual"), "contractual_basis") + self.assertEqual(xform_framework_basis("regulatory"), "regulatory_basis") + self.assertEqual(xform_framework_basis("tax"), "tax_basis") + self.assertEqual(xform_framework_basis("other"), "other_basis") + + def test_basis_with_upper_cases(self): + """Test that the function returns the correct results for each basis in upper cases.""" + self.assertEqual(xform_framework_basis("CASH"), "cash_basis") + self.assertEqual( + xform_framework_basis("OTHER"), + "other_basis", + ) + self.assertEqual(xform_framework_basis("REGULATORY"), "regulatory_basis") + self.assertEqual(xform_framework_basis("TAX"), "tax_basis") + self.assertEqual( + xform_framework_basis("CONTRACTUAL"), + "contractual_basis", + ) + + def test_invalid_basis(self): + """Test that the function raises an exception when the basis is invalid.""" + with self.assertRaises(DataMigrationError): + xform_framework_basis("equity") + + def test_empty_string(self): + """Test that the function raises an exception when the basis is an empty string.""" + with self.assertRaises(DataMigrationError): + xform_framework_basis("") + + def test_none_input(self): + """Test that the function raises an exception when the basis is None.""" + with self.assertRaises(DataMigrationError): + xform_framework_basis(None) + + def test_basis_with_multiple_spaces(self): + """Test that the function returns the correct results when the basis has multiple spaces.""" + self.assertEqual(xform_framework_basis(" cash "), "cash_basis") + self.assertEqual( + xform_framework_basis(" other "), + "other_basis", + ) + with self.assertRaises(DataMigrationError): + xform_framework_basis(" unknown ") + + def test_basis_with_extra_words(self): + """Test that the function raises an exception when the basis has extra words.""" + with self.assertRaises(DataMigrationError): + xform_framework_basis("Something with cash basis") diff --git a/backend/census_historical_migration/test_core_xforms.py b/backend/census_historical_migration/test_core_xforms.py index 06a3622bcc..ec328d9757 100644 --- a/backend/census_historical_migration/test_core_xforms.py +++ b/backend/census_historical_migration/test_core_xforms.py @@ -1,48 +1,194 @@ from django.test import SimpleTestCase -from .exception_utils import DataMigrationError -from .workbooklib.notes_to_sefa import xform_is_minimis_rate_used +from .transforms.xform_string_to_int import ( + string_to_int, +) +from .transforms.xform_string_to_string import string_to_string +from .transforms.xform_string_to_bool import string_to_bool -class TestXformIsMinimisRateUsed(SimpleTestCase): - def test_rate_used(self): - """Test that the function returns 'Y' when the rate is used.""" +from .base_field_maps import FormFieldInDissem, FormFieldMap + +from .sac_general_lib.utils import ( + create_json_from_db_object, + normalize_year_string_or_exit, +) + + +class TestCreateJsonFromDbObject(SimpleTestCase): + """Tests for the create_json_from_db_object function.""" + + class MockDBObject: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + mappings = [ + FormFieldMap( + "first_field_name", "IN_DB_FIRST_FIELD", FormFieldInDissem, "", str + ), + FormFieldMap( + "second_field_name", "IN_DB_SECOND_FIELD", FormFieldInDissem, None, bool + ), + FormFieldMap("third_field_name", None, FormFieldInDissem, "default_value", str), + ] + + def test_normal_case(self): + """Test that the function returns the correct JSON.""" + db_obj = self.MockDBObject(IN_DB_FIRST_FIELD="John", IN_DB_SECOND_FIELD="Y") + expected_json = { + "first_field_name": "John", + "second_field_name": True, + "third_field_name": "default_value", + } self.assertEqual( - # Ensure extra whitespace is acceptable - xform_is_minimis_rate_used("The auditee used the de minimis cost rate."), - "Y", + create_json_from_db_object(db_obj, self.mappings), expected_json ) + def test_missing_field(self): + """ + Test the function for correct JSON output when a database field is missing and the mapping has no default value. + """ + # In this case, second_field_name is not returned because the mapping for this field has no default value. + # and the value of the database field `IN_DB_SECOND_FIELD` is missing. + # `third_field_name` is returned because it has a default value of `default_value` + db_obj = self.MockDBObject(IN_DB_FIRST_FIELD="John") + expected_json = { + "first_field_name": "John", + "third_field_name": "default_value", + } self.assertEqual( - xform_is_minimis_rate_used( - "The School has elected to use the 10-percent de minimis indirect cost rate as allowed under the Uniform Guidance." - ), - "Y", + create_json_from_db_object(db_obj, self.mappings), expected_json ) - def test_rate_not_used(self): - """Test that the function returns 'N' when the rate is not used.""" + def test_none_value(self): + """Test the function for correct JSON output when a database field has a value of None.""" + db_obj = self.MockDBObject(IN_DB_FIRST_FIELD=None, IN_DB_SECOND_FIELD=None) + # In this case, second_field_name is not returned because the mapping for this field has no default value. + # and the value of the database field `IN_DB_SECOND_FIELD` is None. + # `third_field_name` is returned because it has a default value of `default_value` + # `first_field_name` is returned because it has a default value of `""`. + expected_json = {"third_field_name": "default_value"} self.assertEqual( - xform_is_minimis_rate_used( - "The auditee did not use the de minimis cost rate." - ), - "N", + create_json_from_db_object(db_obj, self.mappings), expected_json ) + + def test_all_default_values(self): + """Test the function for correct JSON output when all database fields have default values.""" + db_obj = self.MockDBObject() + # In this case, second_field_name is not returned because the mapping for this field has no default value. + # and the value of the database field `IN_DB_SECOND_FIELD` is None. + # `third_field_name` is returned because it has a default value of `default_value` + # `first_field_name` is returned because it has a default value of `""`. + expected_json = {"first_field_name": "", "third_field_name": "default_value"} self.assertEqual( - xform_is_minimis_rate_used( - "The Board has elected not to use the 10 percent de minimus indirect cost as allowed under the Uniform Guidance." - ), - "N", + create_json_from_db_object(db_obj, self.mappings), expected_json ) - def test_ambiguous_or_unclear_raises_exception(self): - """Test that the function raises an exception when rate usage is ambiguous or unclear.""" - with self.assertRaises(DataMigrationError): - xform_is_minimis_rate_used( - "The information regarding the de minimis rate is not clear." - ) - - with self.assertRaises(DataMigrationError): - xform_is_minimis_rate_used( - "It is unknown whether the de minimis rate was applied." - ) + +class TestStringToBool(SimpleTestCase): + """Tests for the string_to_bool function.""" + + def test_valid_boolean_string(self): + self.assertTrue(string_to_bool("Y")) + self.assertTrue(string_to_bool("y")) + self.assertFalse(string_to_bool("N")) + self.assertFalse(string_to_bool("n")) + + def test_string_with_spaces(self): + self.assertTrue(string_to_bool(" Y ")) + self.assertFalse(string_to_bool(" n")) + + def test_empty_string(self): + with self.assertRaises(ValueError): + string_to_bool("") + + def test_non_string_input(self): + with self.assertRaises(ValueError): + string_to_bool(123) + with self.assertRaises(ValueError): + string_to_bool(None) + with self.assertRaises(ValueError): + string_to_bool(["True"]) + with self.assertRaises(ValueError): + string_to_bool(True) + + def test_string_length_more_than_one(self): + with self.assertRaises(ValueError): + string_to_bool("Yes") + with self.assertRaises(ValueError): + string_to_bool("No") + + def test_invalid_single_character_string(self): + with self.assertRaises(ValueError): + string_to_bool("A") + with self.assertRaises(ValueError): + string_to_bool("Z") + + +class TestStringToInt(SimpleTestCase): + """Tests for the string_to_int function.""" + + def test_valid_integer_strings(self): + self.assertEqual(string_to_int("123"), 123) + self.assertEqual(string_to_int("-456"), -456) + self.assertEqual(string_to_int(" 789 "), 789) + + def test_invalid_strings(self): + with self.assertRaises(ValueError): + string_to_int("abc") + with self.assertRaises(ValueError): + string_to_int("123abc") + with self.assertRaises(ValueError): + string_to_int("12.34") + + def test_empty_string(self): + with self.assertRaises(ValueError): + string_to_int("") + + def test_non_string_input(self): + with self.assertRaises(ValueError): + string_to_int(123) + with self.assertRaises(ValueError): + string_to_int(None) + with self.assertRaises(ValueError): + string_to_int([123]) + + +class TestStringToString(SimpleTestCase): + def test_valid_strings(self): + self.assertEqual(string_to_string("hello"), "hello") + self.assertEqual(string_to_string(" world "), "world") + self.assertEqual(string_to_string(" space both sides "), "space both sides") + + def test_none_input(self): + self.assertEqual(string_to_string(None), "") + + def test_non_string_input(self): + with self.assertRaises(ValueError): + string_to_string(123) + with self.assertRaises(ValueError): + string_to_string(True) + with self.assertRaises(ValueError): + string_to_string([1, 2, 3]) + + def test_string_with_only_spaces(self): + self.assertEqual(string_to_string(" "), "") + + +class TestNormalizeYearString(SimpleTestCase): + def test_valid_short_year(self): + self.assertEqual(normalize_year_string_or_exit("21"), "2021") + self.assertEqual(normalize_year_string_or_exit("16"), "2016") + + def test_valid_full_year(self): + self.assertEqual(normalize_year_string_or_exit("2018"), "2018") + + def test_invalid_year_string(self): + with self.assertRaises(SystemExit): + normalize_year_string_or_exit("invalid") + + def test_year_out_of_range(self): + with self.assertRaises(SystemExit): + normalize_year_string_or_exit("15") + with self.assertRaises(SystemExit): + normalize_year_string_or_exit("2023") diff --git a/backend/census_historical_migration/test_excel_creation.py b/backend/census_historical_migration/test_excel_creation.py index 0f8e5044fa..6b23f48e1c 100644 --- a/backend/census_historical_migration/test_excel_creation.py +++ b/backend/census_historical_migration/test_excel_creation.py @@ -63,7 +63,7 @@ def test_set_range_default(self): def test_set_range_no_default(self): """ - Default to empty string when no value or default given + Error when no value or default given """ wb = self.init_named_range("A6") ws = wb.active @@ -71,12 +71,7 @@ def test_set_range_no_default(self): ws.cell(row=6, column=1, value="foo") self.assertEqual(ws["A6"].value, "foo") - set_range( - wb, - self.range_name, - [None], - ) - self.assertEqual(ws["A6"].value, "") + self.assertRaises(ValueError, set_range, wb, self.range_name, [None]) def test_set_range_conversion(self): """ @@ -165,18 +160,19 @@ def test_int_conversion(self): self.assertEqual(apply_conversion_function("123", "default", int), 123) def test_custom_conversion(self): - """Test that a custom conversion function is properly applied""" + """Test that custom conversion function is applied when present""" self.assertEqual( apply_conversion_function("test", "default", lambda x: x.upper()), "TEST" ) def test_default_value(self): """Test that a default value is returned when the input is None""" - self.assertEqual(apply_conversion_function("", "default", str), "default") + self.assertEqual(apply_conversion_function(None, "default", str), "default") def test_none_with_no_default(self): - """Test that an empty string is returned when the input is None and no default is provided""" - self.assertEqual(apply_conversion_function(None, None, str), "") + """Test that an exception is raised when the input is None and no default is provided""" + with self.assertRaises(ValueError): + apply_conversion_function(None, None, str) class TestGetRanges(TestCase): diff --git a/backend/census_historical_migration/test_general_information_xforms.py b/backend/census_historical_migration/test_general_information_xforms.py new file mode 100644 index 0000000000..1d5a93d909 --- /dev/null +++ b/backend/census_historical_migration/test_general_information_xforms.py @@ -0,0 +1,212 @@ +from datetime import datetime, timedelta +from django.test import SimpleTestCase + +from .sac_general_lib.general_information import ( + AUDIT_TYPE_DICT, + PERIOD_DICT, + xform_audit_period_covered, + xform_audit_type, + xform_auditee_fiscal_period_end, + xform_auditee_fiscal_period_start, + xform_country, + xform_entity_type, +) + + +from .exception_utils import DataMigrationError + + +class TestXformEntityType(SimpleTestCase): + def test_valid_phrases(self): + """Test that the function returns the correct results when given valid phrases.""" + self.assertEqual( + xform_entity_type("institution of higher education"), "higher-ed" + ) + self.assertEqual(xform_entity_type("nonprofit"), "non-profit") + self.assertEqual(xform_entity_type("Non-Profit"), "non-profit") + self.assertEqual(xform_entity_type("local government"), "local") + self.assertEqual(xform_entity_type("state"), "state") + self.assertEqual(xform_entity_type("unknown"), "unknown") + self.assertEqual(xform_entity_type("tribe"), "tribal") + self.assertEqual(xform_entity_type("tribal"), "tribal") + + def test_valid_phrases_with_extra_whitespace(self): + """Test that the function returns the correct results when given valid phrases with extra whitespace.""" + self.assertEqual( + xform_entity_type("institution of higher education "), "higher-ed" + ) + self.assertEqual(xform_entity_type(" nonprofit "), "non-profit") + self.assertEqual(xform_entity_type("Non-Profit "), "non-profit") + self.assertEqual(xform_entity_type(" local government "), "local") + self.assertEqual(xform_entity_type(" state "), "state") + + def test_valid_phrases_with_extra_whitespace_with_capitalization(self): + """Test that the function returns the correct results when given valid phrases with extra whitespace and upper cases.""" + self.assertEqual( + xform_entity_type("Institution of Higher Education "), "higher-ed" + ) + self.assertEqual(xform_entity_type(" Nonprofit "), "non-profit") + self.assertEqual(xform_entity_type("Non-Profit "), "non-profit") + self.assertEqual(xform_entity_type(" LOCAL Government "), "local") + self.assertEqual(xform_entity_type(" STATE "), "state") + self.assertEqual(xform_entity_type(" Tribal "), "tribal") + + def test_valid_phrases_with_extra_whitespace_with_extra_words_with_capitalization( + self, + ): + """Test that the function returns the correct results when given valid phrases with extra whitespace and extra words.""" + self.assertEqual( + xform_entity_type("Institution of Higher Education (IHE) and Research"), + "higher-ed", + ) + + self.assertEqual( + xform_entity_type(" LOCAL GOVERNMENT AND AGENCIES "), + "local", + ) + self.assertEqual( + xform_entity_type(" STATE and Non-Governmental Organization "), "state" + ) + self.assertEqual( + xform_entity_type(" Indian tribe or Tribal Organization "), "tribal" + ) + + def test_invalid_phrase(self): + """Test that the function raises an exception when given an invalid phrase.""" + with self.assertRaises(DataMigrationError): + xform_entity_type("business") + + def test_empty_string(self): + """Test that the function raises an exception when given an empty string.""" + with self.assertRaises(DataMigrationError): + xform_entity_type("") + + def test_none_input(self): + """Test that the function raises an exception when given None.""" + with self.assertRaises(DataMigrationError): + xform_entity_type(None) + + +class TestXformCountry(SimpleTestCase): + class MockAuditHeader: + def __init__(self, CPASTATE): + self.CPASTATE = CPASTATE + + def setUp(self): + self.general_information = { + "auditor_country": "", + } + self.audit_header = self.MockAuditHeader("") + + def test_when_auditor_country_set_to_us(self): + """Test that the function returns the correct results when the auditor country is set to US.""" + self.general_information["auditor_country"] = "US" + result = xform_country(self.general_information, self.audit_header) + self.assertEqual(result["auditor_country"], "USA") + + def test_when_auditor_country_set_to_usa(self): + """Test that the function returns the correct results when the auditor country is set to USA.""" + self.general_information["auditor_country"] = "USA" + result = xform_country(self.general_information, self.audit_header) + self.assertEqual(result["auditor_country"], "USA") + + def test_when_auditor_country_set_to_empty_string_and_auditor_state_valid(self): + """Test that the function returns the correct results when the auditor country is set to an empty string.""" + self.general_information["auditor_country"] = "" + self.audit_header.CPASTATE = "MA" + result = xform_country(self.general_information, self.audit_header) + self.assertEqual(result["auditor_country"], "USA") + + def test_when_auditor_country_set_to_empty_string_and_auditor_state_invalid(self): + """Test that the function raises an exception when the auditor country is set to an empty string and the auditor state is invalid.""" + self.general_information["auditor_country"] = "" + self.audit_header.CPASTATE = "XX" + with self.assertRaises(DataMigrationError): + xform_country(self.general_information, self.audit_header) + + +class TestXformAuditeeFiscalPeriodEnd(SimpleTestCase): + def setUp(self): + self.general_information = { + "auditee_fiscal_period_end": "01/31/2021 00:00:00", + } + + def test_when_auditee_fiscal_period_end_is_valid(self): + """Test that the function returns the correct results when the fiscal period end is valid.""" + result = xform_auditee_fiscal_period_end(self.general_information) + self.assertEqual(result["auditee_fiscal_period_end"], "2021-01-31") + + def test_when_auditee_fiscal_period_end_is_invalid(self): + """Test that the function raises an exception when the fiscal period end is invalid.""" + self.general_information["auditee_fiscal_period_end"] = "01/31/2021" + with self.assertRaises(ValueError): + xform_auditee_fiscal_period_end(self.general_information) + self.general_information["auditee_fiscal_period_end"] = "" + with self.assertRaises(DataMigrationError): + xform_auditee_fiscal_period_end(self.general_information) + + +class TestXformAuditeeFiscalPeriodStart(SimpleTestCase): + def setUp(self): + self.general_information = { + "auditee_fiscal_period_end": "01/31/2021 00:00:00", + } + + def test_when_auditee_fiscal_period_end_is_valid(self): + """Test that the function returns the correct results when the fiscal period end is valid.""" + result = xform_auditee_fiscal_period_start(self.general_information) + expected_date = ( + datetime.strptime( + self.general_information["auditee_fiscal_period_end"], + "%m/%d/%Y %H:%M:%S", + ) + - timedelta(days=365) + ).strftime("%Y-%m-%d") + self.assertEqual(result["auditee_fiscal_period_start"], expected_date) + + def test_when_auditee_fiscal_period_end_is_invalid(self): + """Test that the function raises an exception when the fiscal period end is invalid.""" + self.general_information["auditee_fiscal_period_end"] = "01/31/2021" + with self.assertRaises(ValueError): + xform_auditee_fiscal_period_start(self.general_information) + self.general_information["auditee_fiscal_period_end"] = "" + with self.assertRaises(ValueError): + xform_auditee_fiscal_period_start(self.general_information) + + +class TestXformAuditPeriodCovered(SimpleTestCase): + def test_valid_period(self): + for key, value in PERIOD_DICT.items(): + with self.subTest(key=key): + general_information = {"audit_period_covered": key} + result = xform_audit_period_covered(general_information) + self.assertEqual(result["audit_period_covered"], value) + + def test_invalid_period(self): + general_information = {"audit_period_covered": "invalid_key"} + with self.assertRaises(DataMigrationError): + xform_audit_period_covered(general_information) + + def test_missing_period(self): + general_information = {} + with self.assertRaises(DataMigrationError): + xform_audit_period_covered(general_information) + + +class TestXformAuditType(SimpleTestCase): + def test_valid_audit_type(self): + for key, value in AUDIT_TYPE_DICT.items(): + with self.subTest(key=key): + general_information = {"audit_type": key} + result = xform_audit_type(general_information) + self.assertEqual(result["audit_type"], value) + + def test_invalid_audit_type(self): + general_information = {"audit_type": "invalid_key"} + with self.assertRaises(DataMigrationError): + xform_audit_type(general_information) + + def test_missing_audit_type(self): + general_information = {} + with self.assertRaises(DataMigrationError): + xform_audit_type(general_information) diff --git a/backend/census_historical_migration/test_notes_to_sefa_xforms.py b/backend/census_historical_migration/test_notes_to_sefa_xforms.py new file mode 100644 index 0000000000..be8ef83aea --- /dev/null +++ b/backend/census_historical_migration/test_notes_to_sefa_xforms.py @@ -0,0 +1,103 @@ +from django.test import SimpleTestCase + +from .exception_utils import DataMigrationError +from .workbooklib.notes_to_sefa import xform_is_minimis_rate_used + + +class TestXformIsMinimisRateUsed(SimpleTestCase): + def test_rate_used(self): + """Test that the function returns 'Y' when the rate is used.""" + self.assertEqual( + xform_is_minimis_rate_used("The auditee used the de minimis cost rate."), + "Y", + ) + + self.assertEqual( + xform_is_minimis_rate_used( + "The School has elected to use the 10-percent de minimis indirect cost rate as allowed under the Uniform Guidance." + ), + "Y", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "They have used the de minimis rate for this project." + ), + "Y", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "The auditee organization elected to use the de minimis rate." + ), + "Y", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "The de minimis rate is used and is allowed under our policy." + ), + "Y", + ) + + def test_rate_not_used(self): + """Test that the function returns 'N' when the rate is not used.""" + self.assertEqual( + xform_is_minimis_rate_used( + "The auditee did not use the de minimis cost rate." + ), + "N", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "The Board has elected not to use the 10 percent de minimus indirect cost as allowed under the Uniform Guidance." + ), + "N", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "The organization did not use the de minimis rate." + ), + "N", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "It was decided not to use the de minimis rate in this case." + ), + "N", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "The institution has elected not to use the de minimis rate." + ), + "N", + ) + + def test_rate_with_multiple_spaces(self): + """Test that the function returns the correct results when the rate is used and there are multiple spaces between words.""" + self.assertEqual( + xform_is_minimis_rate_used( + "We have elected to use the de minimis rate." + ), + "Y", + ) + self.assertEqual( + xform_is_minimis_rate_used( + "The organization did not use the de minimis rate." + ), + "N", + ) + + def test_ambiguous_or_unclear_raises_exception(self): + """Test that the function raises an exception when rate usage is ambiguous or unclear.""" + with self.assertRaises(DataMigrationError): + xform_is_minimis_rate_used( + "The information regarding the de minimis rate is not clear." + ) + + with self.assertRaises(DataMigrationError): + xform_is_minimis_rate_used( + "It is unknown whether the de minimis rate was applied." + ) + + def test_empty_string(self): + """Test that the function raises an exception when the input is an empty string.""" + with self.assertRaises(DataMigrationError): + xform_is_minimis_rate_used("") diff --git a/backend/census_historical_migration/transforms/xform_string_to_bool.py b/backend/census_historical_migration/transforms/xform_string_to_bool.py index a866f4c5c5..c4bcac36a4 100644 --- a/backend/census_historical_migration/transforms/xform_string_to_bool.py +++ b/backend/census_historical_migration/transforms/xform_string_to_bool.py @@ -1,14 +1,11 @@ def string_to_bool(value): """Converts a string to a boolean.""" - if isinstance(value, bool): - return value - if not isinstance(value, str): raise ValueError(f"Expected string, got {type(value).__name__}") - value = value.strip() - if len(value) > 1: - raise ValueError(f"Expected string of length 1, got {len(value)}") + new_value = value.strip().upper() + if new_value not in ["Y", "N"]: + raise ValueError(f"Expected 'Y' or 'N', got '{value}'") - return value.upper() == "Y" + return new_value == "Y" diff --git a/backend/census_historical_migration/transforms/xform_string_to_int.py b/backend/census_historical_migration/transforms/xform_string_to_int.py index 722da27174..b263ddf736 100644 --- a/backend/census_historical_migration/transforms/xform_string_to_int.py +++ b/backend/census_historical_migration/transforms/xform_string_to_int.py @@ -1,12 +1,9 @@ def string_to_int(value): """Converts a string to an integer.""" - if isinstance(value, int): - return value + if not isinstance(value, str): raise ValueError(f"Expected string, got {type(value).__name__}") - value = value.strip() - # Check if the string can be converted to an integer try: return int(value) diff --git a/backend/census_historical_migration/transforms/xform_string_to_string.py b/backend/census_historical_migration/transforms/xform_string_to_string.py index f947097f3d..ff74b19265 100644 --- a/backend/census_historical_migration/transforms/xform_string_to_string.py +++ b/backend/census_historical_migration/transforms/xform_string_to_string.py @@ -1,13 +1,11 @@ def string_to_string(value): """ Converts a string to a trimmed string. Returns an empty string if the input - is empty or 'nan'.""" + is None.""" if value is None: return "" if not isinstance(value, str): raise ValueError(f"Expected string, got {type(value).__name__}") trimmed_value = value.strip() - # FIXME-MSHD: When some CSV files are loaded - # to Postgres DB, empty string are being converted into 'nan' - # This is a temporary fix to handle the issue, we need to investigate this further. - return "" if trimmed_value in ["nan"] else trimmed_value + + return trimmed_value diff --git a/backend/census_historical_migration/workbooklib/additional_eins.py b/backend/census_historical_migration/workbooklib/additional_eins.py index 53554c6416..8357342f81 100644 --- a/backend/census_historical_migration/workbooklib/additional_eins.py +++ b/backend/census_historical_migration/workbooklib/additional_eins.py @@ -24,10 +24,19 @@ def xform_remove_trailing_decimal_zero(value): """ Removes trailing .0 from a EIN strings. + Raises an exception if the decimal part is not 0. """ trimmed_ein = string_to_string(value) - if trimmed_ein.endswith(".0"): - return trimmed_ein[:-2] + + if "." in trimmed_ein: + whole, decimal = trimmed_ein.split(".") + if decimal == "0": + return whole + else: + raise ValueError( + f"additional_ein has non zero decimal value: {trimmed_ein}" + ) + return trimmed_ein @@ -53,9 +62,9 @@ def generate_additional_eins(audit_header, outfile): logger.info( f"--- generate additional eins {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" ) - + uei = string_to_string(audit_header.UEI) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_EINS]) - set_workbook_uei(wb, audit_header.UEI) + set_workbook_uei(wb, uei) addl_eins = _get_eins(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, addl_eins) wb.save(outfile) @@ -63,5 +72,5 @@ def generate_additional_eins(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "additional_eins", mappings, addl_eins ) - table["singletons"]["auditee_uei"] = audit_header.UEI + table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/additional_ueis.py b/backend/census_historical_migration/workbooklib/additional_ueis.py index f0180c96f1..b1f92ac691 100644 --- a/backend/census_historical_migration/workbooklib/additional_ueis.py +++ b/backend/census_historical_migration/workbooklib/additional_ueis.py @@ -1,3 +1,4 @@ +from ..transforms.xform_string_to_string import string_to_string from ..workbooklib.excel_creation_utils import ( map_simple_columns, generate_dissemination_test_table, @@ -18,7 +19,6 @@ logger = logging.getLogger(__name__) mappings = [ - # FIXME: We have no dissemination nodel for this. SheetFieldMap("additional_uei", "UEI", WorkbookFieldInDissem, None, str), ] @@ -34,9 +34,9 @@ def generate_additional_ueis(audit_header, outfile): logger.info( f"--- generate additional ueis {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" ) - + uei = string_to_string(audit_header.UEI) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_UEIS]) - set_workbook_uei(wb, audit_header.UEI) + set_workbook_uei(wb, uei) additional_ueis = _get_ueis(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, additional_ueis) wb.save(outfile) @@ -44,5 +44,5 @@ def generate_additional_ueis(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "additional_ueis", mappings, additional_ueis ) - table["singletons"]["auditee_uei"] = audit_header.UEI + table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/corrective_action_plan.py b/backend/census_historical_migration/workbooklib/corrective_action_plan.py index 77b874e1fb..8431b05ff6 100644 --- a/backend/census_historical_migration/workbooklib/corrective_action_plan.py +++ b/backend/census_historical_migration/workbooklib/corrective_action_plan.py @@ -1,3 +1,4 @@ +from ..transforms.xform_string_to_string import string_to_string from ..workbooklib.excel_creation_utils import ( map_simple_columns, generate_dissemination_test_table, @@ -41,10 +42,11 @@ def generate_corrective_action_plan(audit_header, outfile): f"--- generate corrective action plan {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" ) + uei = string_to_string(audit_header.UEI) wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.CORRECTIVE_ACTION_PLAN] ) - set_workbook_uei(wb, audit_header.UEI) + set_workbook_uei(wb, uei) captexts = _get_cap_text(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, captexts) wb.save(outfile) @@ -52,5 +54,5 @@ def generate_corrective_action_plan(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "corrective_action_plans", mappings, captexts ) - table["singletons"]["auditee_uei"] = audit_header.UEI + table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/excel_creation_utils.py b/backend/census_historical_migration/workbooklib/excel_creation_utils.py index 70bca2de32..5268cdbf77 100644 --- a/backend/census_historical_migration/workbooklib/excel_creation_utils.py +++ b/backend/census_historical_migration/workbooklib/excel_creation_utils.py @@ -1,3 +1,6 @@ +from ..transforms.xform_string_to_bool import ( + string_to_bool, +) from ..transforms.xform_string_to_string import ( string_to_string, ) @@ -19,7 +22,6 @@ column_index_from_string, ) -import sys import logging @@ -78,15 +80,20 @@ def apply_conversion_function(value, default, conversion_function): """ Helper to apply a conversion function to a value, or use a default value """ - if value: - if conversion_function is str: - new_value = string_to_string(value) - elif conversion_function is int: - new_value = string_to_int(value) - else: - new_value = conversion_function(value) + if value is None and default is None: + raise ValueError("No value or default provided") + + selected_value = value if value is not None else default + + if conversion_function is str: + new_value = string_to_string(selected_value) + elif conversion_function is int: + new_value = string_to_int(selected_value) + elif conversion_function is bool: + new_value = string_to_bool(selected_value) else: - new_value = default or "" + new_value = conversion_function(selected_value) + return new_value @@ -154,7 +161,9 @@ def map_simple_columns(wb, mappings, values): list(map(lambda m: m.in_sheet, mappings)) ) ) - sys.exit(-1) + raise DataMigrationError( + "Invaid mappings. You repeated a field in the mappings" + ) # Map all the simple ones for m in mappings: @@ -211,3 +220,19 @@ def generate_dissemination_test_table(audit_header, api_endpoint, mappings, obje def get_audits(dbkey, year): """Returns Audits records for the given dbkey and audit year.""" return Audits.objects.filter(DBKEY=dbkey, AUDITYEAR=year).order_by("ID") + + +def xform_add_hyphen_to_zip(zip): + """ + Transform a ZIP code string by adding a hyphen. If the ZIP code has 9 digits, inserts a hyphen after the fifth digit. + Returns the original ZIP code if it has 5 digits or is malformed. + """ + strzip = string_to_string(zip) + if len(strzip) == 5: + return strzip + elif len(strzip) == 9: + # FIXME - MSHD: This is a transformation and might require logging. + return f"{strzip[0:5]}-{strzip[5:9]}" + else: + # FIXME - MSHD: How do we handle 4-digit and 8-digit ZIP codes? + raise DataMigrationError("Zip code is malformed in secondary auditor.") diff --git a/backend/census_historical_migration/workbooklib/federal_awards.py b/backend/census_historical_migration/workbooklib/federal_awards.py index 710d829007..968a221056 100644 --- a/backend/census_historical_migration/workbooklib/federal_awards.py +++ b/backend/census_historical_migration/workbooklib/federal_awards.py @@ -51,7 +51,7 @@ def if_zero_empty(v): "state_cluster_name", "STATECLUSTERNAME", WorkbookFieldInDissem, None, str ), SheetFieldMap( - "federal_program_total", "PROGRAMTOTAL", WorkbookFieldInDissem, 0, int + "federal_program_total", "PROGRAMTOTAL", WorkbookFieldInDissem, None, int ), SheetFieldMap( "additional_award_identification", @@ -60,7 +60,7 @@ def if_zero_empty(v): None, str, ), - SheetFieldMap("cluster_total", "CLUSTERTOTAL", WorkbookFieldInDissem, 0, int), + SheetFieldMap("cluster_total", "CLUSTERTOTAL", WorkbookFieldInDissem, None, int), SheetFieldMap("is_guaranteed", "LOANS", "is_loan", None, str), # In the intake process, we initially use convert_to_stripped_string to convert IR values into strings, # and then apply specific functions like convert_loan_balance_to_integers_or_na to convert particular fields @@ -78,14 +78,14 @@ def if_zero_empty(v): "PASSTHROUGHAMOUNT", "passthrough_amount", None, - if_zero_empty, + if_zero_empty, # FIXME - MSHD: This according to ticket #2912 ), SheetFieldMap("is_major", "MAJORPROGRAM", WorkbookFieldInDissem, None, str), SheetFieldMap("audit_report_type", "TYPEREPORT_MP", "audit_report_type", None, str), SheetFieldMap( - "number_of_audit_findings", "FINDINGSCOUNT", "findings_count", 0, int + "number_of_audit_findings", "FINDINGSCOUNT", "findings_count", None, int ), - SheetFieldMap("amount_expended", "AMOUNT", WorkbookFieldInDissem, 0, int), + SheetFieldMap("amount_expended", "AMOUNT", WorkbookFieldInDissem, None, int), ] @@ -103,8 +103,10 @@ def _generate_cluster_names( audits: list[Audits], ) -> tuple[list[str], list[str], list[str]]: """Reconstructs the cluster names for each audit in the provided list.""" - # Patch the clusternames. They used to be allowed to enter anything - # they wanted. + # FIXME - MSHD: For the sake of data migration, we will: + # 1. Remove the cluster name validation logic from this code. + # 2. Remove cluster name validation from json schema. + # 3. Implement cluster name validation in python against the IR and exclude census data from this logic. valid_file = open(f"{settings.BASE_DIR}/schemas/source/base/ClusterNames.json") valid_json = json.load(valid_file) cluster_names = [] @@ -144,7 +146,10 @@ def _get_full_cfdas(audits): and CFDA_EXT attributes of each audit object, separated by a dot. """ # audit.CFDA is not used here because it does not always match f"{audit.CFDA_PREFIX}.{audit.CFDA_EXT}" - return [f"{audit.CFDA_PREFIX}.{audit.CFDA_EXT}" for audit in audits] + return [ + f"{string_to_string(audit.CFDA_PREFIX)}.{string_to_string(audit.CFDA_EXT)}" + for audit in audits + ] # The functionality of _fix_passthroughs has been split into two separate functions: @@ -199,9 +204,10 @@ def _xform_populate_default_passthrough_values( for index, audit, name, id in zip( range(len(audits)), audits, passthrough_names, passthrough_ids ): - if audit.DIRECT == "N" and name == "": + direct = string_to_string(audit.DIRECT) + if direct == "N" and name == "": passthrough_names[index] = "NO PASSTHROUGH NAME PROVIDED" - if audit.DIRECT == "N" and id == "": + if direct == "N" and id == "": passthrough_ids[index] = "NO PASSTHROUGH ID PROVIDED" return (passthrough_names, passthrough_ids) @@ -216,14 +222,16 @@ def _xform_populate_default_loan_balance(loans_at_end, audits): If the audit's LOANS attribute is "Y" and the loan balance is empty, it fills in a default value indicating that no loan balance was provided.""" for ndx, audit in zip(range(len(audits)), audits, loans_at_end): - if audit.LOANS == "Y": - if audit.LOANBALANCE is None: + loan = string_to_string(audit.LOANS).upper() + balance = string_to_string(audit.LOANBALANCE) + if loan == "Y": + if not balance: loans_at_end[ ndx ] = 1 # FIXME - MSHD: This value requires team approval. # There are cases (dbkeys 148665/150450) with balance = -1.0, how do we handle this? else: - if audit.LOANBALANCE is not None: + if not balance: loans_at_end[ndx] = "" return loans_at_end @@ -266,7 +274,8 @@ def generate_federal_awards(audit_header, outfile): wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED] ) - set_workbook_uei(wb, audit_header.UEI) + uei = string_to_string(audit_header.UEI) + set_workbook_uei(wb, uei) audits = get_audits(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, audits) @@ -364,7 +373,7 @@ def generate_federal_awards(audit_header, outfile): award["values"].append(other_cluster_name) award_counter += 1 - table["singletons"]["auditee_uei"] = audit_header.UEI + table["singletons"]["auditee_uei"] = uei table["singletons"]["total_amount_expended"] = total return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/findings.py b/backend/census_historical_migration/workbooklib/findings.py index 2fd4bc59f1..58c0f4f7a6 100644 --- a/backend/census_historical_migration/workbooklib/findings.py +++ b/backend/census_historical_migration/workbooklib/findings.py @@ -27,19 +27,19 @@ def sorted_string(s): def xform_prior_year_findings(value): """ - Transform the value of prior_references to N/A if empty.""" + Transform the value of prior_references to N/A if empty. + """ trimmed_value = string_to_string(value) - if trimmed_value == "": + if not trimmed_value: # FIXME - MSHD: This is a transformation and might require logging. # Why is this transformation needed? Because users were allowed to leave this empty # but we have decided to enforce that they enter N/A (starting in 2023). # Therefore, we need to transform the empty string to N/A, otherwise the new validation # rule will fail most of the migration. # logger.info(f"Prior year findings is empty. Setting to N/A.") - new_value = "N/A" - else: - new_value = trimmed_value - return new_value + return "N/A" + + return trimmed_value mappings = [ @@ -133,7 +133,8 @@ def generate_findings(audit_header, outfile): wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE] ) - set_workbook_uei(wb, audit_header.UEI) + uei = string_to_string(audit_header.UEI) + set_workbook_uei(wb, uei) audits = get_audits(audit_header.DBKEY, audit_header.AUDITYEAR) # For each of them, I need to generate an elec -> award mapping. e2a = {} diff --git a/backend/census_historical_migration/workbooklib/findings_text.py b/backend/census_historical_migration/workbooklib/findings_text.py index 78a409c1c0..71b2ea22bf 100644 --- a/backend/census_historical_migration/workbooklib/findings_text.py +++ b/backend/census_historical_migration/workbooklib/findings_text.py @@ -1,3 +1,4 @@ +from ..transforms.xform_string_to_string import string_to_string from ..workbooklib.excel_creation_utils import ( map_simple_columns, generate_dissemination_test_table, @@ -43,7 +44,8 @@ def generate_findings_text(audit_header, outfile): ) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.FINDINGS_TEXT]) - set_workbook_uei(wb, audit_header.UEI) + uei = string_to_string(audit_header.UEI) + set_workbook_uei(wb, uei) findings_texts = _get_findings_texts(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, findings_texts) @@ -53,6 +55,6 @@ def generate_findings_text(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "findings_text", mappings, findings_texts ) - table["singletons"]["auditee_uei"] = audit_header.UEI + table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py index 2553ab00ca..829d379754 100644 --- a/backend/census_historical_migration/workbooklib/notes_to_sefa.py +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -122,8 +122,8 @@ def generate_notes_to_sefa(audit_header, outfile): ) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.NOTES_TO_SEFA]) - - set_workbook_uei(wb, audit_header.UEI) + uei = string_to_string(audit_header.UEI) + set_workbook_uei(wb, uei) notes = _get_notes(audit_header.DBKEY, audit_header.AUDITYEAR) rate_content = _get_minimis_cost_rate(audit_header.DBKEY, audit_header.AUDITYEAR) policies_content = _get_accounting_policies( @@ -152,6 +152,6 @@ def generate_notes_to_sefa(audit_header, outfile): table["singletons"]["accounting_policies"] = policies_content table["singletons"]["is_minimis_rate_used"] = is_minimis_rate_used table["singletons"]["rate_explained"] = rate_content - table["singletons"]["auditee_uei"] = audit_header.UEI + table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/post_upload_utils.py b/backend/census_historical_migration/workbooklib/post_upload_utils.py index c752f1cf1a..d7e11d089f 100644 --- a/backend/census_historical_migration/workbooklib/post_upload_utils.py +++ b/backend/census_historical_migration/workbooklib/post_upload_utils.py @@ -1,8 +1,3 @@ -"""Fixtures for SingleAuditChecklist. - -We want to create a variety of SACs in different states of -completion. -""" import logging from pathlib import Path @@ -87,16 +82,16 @@ def _post_upload_pdf(this_sac, this_user, pdf_filename): print(file.__dict__) pdf_file = PDFFile( file=file, - component_page_numbers={ # FIXME - How do we want to handle these values? - "financial_statements": 1, - "financial_statements_opinion": 2, - "schedule_expenditures": 3, - "schedule_expenditures_opinion": 4, - "uniform_guidance_control": 5, - "uniform_guidance_compliance": 6, - "GAS_control": 6, - "GAS_compliance": 7, - "schedule_findings": 8, + component_page_numbers={ # FIXME MSHD- see ticket #2912 + "financial_statements": -1, + "financial_statements_opinion": -1, + "schedule_expenditures": -1, + "schedule_expenditures_opinion": -1, + "uniform_guidance_control": -1, + "uniform_guidance_compliance": -1, + "GAS_control": -1, + "GAS_compliance": -1, + "schedule_findings": -1, }, filename=Path(pdf_filename).stem, user=this_user, @@ -111,33 +106,10 @@ def _post_upload_pdf(this_sac, this_user, pdf_filename): this_sac.save() -def _post_upload_workbook(this_sac, this_user, section, xlsx_file): - """Upload a workbook for this SAC. - - This should be idempotent if it is called on a SAC that already - has a federal awards file uploaded. - """ - ExcelFile = apps.get_model("audit.ExcelFile") - - if ( - ExcelFile.objects.filter(sac_id=this_sac.id, form_section=section).exists() - and get_field_by_section(this_sac, section) is not None - ): - # there is already an uploaded file and data in the object so - # nothing to do here - return - - excel_file = ExcelFile( - file=xlsx_file, - filename=Path("xlsx.xlsx").stem, - user=this_user, - sac_id=this_sac.id, - form_section=section, - ) - excel_file.full_clean() - excel_file.save() +def _post_upload_workbook(this_sac, section, xlsx_file): + """Upload a workbook for this SAC.""" - audit_data = extract_mapping[section](excel_file.file) + audit_data = extract_mapping[section](xlsx_file) validator_mapping[section](audit_data) if section == FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: diff --git a/backend/census_historical_migration/workbooklib/secondary_auditors.py b/backend/census_historical_migration/workbooklib/secondary_auditors.py index a98a03f49d..a8d1222975 100644 --- a/backend/census_historical_migration/workbooklib/secondary_auditors.py +++ b/backend/census_historical_migration/workbooklib/secondary_auditors.py @@ -3,6 +3,7 @@ map_simple_columns, generate_dissemination_test_table, set_workbook_uei, + xform_add_hyphen_to_zip, ) from ..base_field_maps import SheetFieldMap from ..workbooklib.templates import sections_to_template_paths @@ -16,22 +17,6 @@ logger = logging.getLogger(__name__) -def xform_add_hyphen_to_zip(zip): - """ - Transform a ZIP code string by adding a hyphen. If the ZIP code has 9 digits, inserts a hyphen after the fifth digit. - Returns the original ZIP code if it has 5 digits or is malformed. - """ - strzip = string_to_string(zip) - if len(strzip) == 5: - return strzip - elif len(strzip) == 9: - # FIXME - MSHD: This is a transformation and might require logging. - return f"{strzip[0:5]}-{strzip[5:9]}" - else: - logger.info("ZIP IS MALFORMED IN WORKBOOKS E2E / SAC_CREATION") - return strzip - - mappings = [ SheetFieldMap( "secondary_auditor_address_city", "CPACITY", "address_city", None, str @@ -89,7 +74,8 @@ def generate_secondary_auditors(audit_header, outfile): wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.SECONDARY_AUDITORS] ) - set_workbook_uei(wb, audit_header.UEI) + uei = string_to_string(audit_header.UEI) + set_workbook_uei(wb, uei) secondary_auditors = _get_secondary_auditors( audit_header.DBKEY, audit_header.AUDITYEAR ) @@ -99,6 +85,6 @@ def generate_secondary_auditors(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "secondary_auditors", mappings, secondary_auditors ) - table["singletons"]["auditee_uei"] = audit_header.UEI + table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/workbook_builder_loader.py b/backend/census_historical_migration/workbooklib/workbook_builder_loader.py index 8ab5b021cc..70ddce180d 100644 --- a/backend/census_historical_migration/workbooklib/workbook_builder_loader.py +++ b/backend/census_historical_migration/workbooklib/workbook_builder_loader.py @@ -21,7 +21,7 @@ def _loader(workbook_generator, section): ) if user: - _post_upload_workbook(sac, user, section, excel_file) + _post_upload_workbook(sac, section, excel_file) else: raise Exception("User must be provided to upload workbook") From b8222adf3ca384438179b0c9c69ba82891c4d41b Mon Sep 17 00:00:00 2001 From: Tadhg O'Higgins <2626258+tadhg-ohiggins@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:48:07 -0800 Subject: [PATCH 2/4] Add search fields to User and UserPermission in Django Admin. (#2996) --- backend/users/admin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/users/admin.py b/backend/users/admin.py index d5c00f3b00..59cf6690f0 100644 --- a/backend/users/admin.py +++ b/backend/users/admin.py @@ -20,6 +20,7 @@ class UserAdmin(admin.ModelAdmin): list_display = ["email", "can_read_tribal", "last_login", "date_joined"] exclude = ["groups", "user_permissions", "password"] readonly_fields = ["date_joined", "last_login"] + search_fields = ("email", "username") def can_read_tribal(self, obj): return _can_read_tribal(obj) @@ -28,6 +29,7 @@ def can_read_tribal(self, obj): @admin.register(UserPermission) class UserPermissionAdmin(admin.ModelAdmin): list_display = ["user", "email", "permission"] + search_fields = ("email", "permission", "user") @admin.register(StaffUserLog) From 9f59996de01c10c91afd3e0652fbd1b078599ec7 Mon Sep 17 00:00:00 2001 From: Tadhg O'Higgins <2626258+tadhg-ohiggins@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:48:55 -0800 Subject: [PATCH 3/4] Add page/backend for adding editors to submission (#2994) * Add backend functionality and unlinked first-draft page for adding editors to a submission. * Header correction. --- .../manage-submission-change-access.html | 28 +++--- .../test_manage_submission_access_view.py | 88 +++++++++++++++++++ backend/audit/urls.py | 5 ++ backend/audit/views/__init__.py | 2 + .../audit/views/manage_submission_access.py | 35 +++++--- 5 files changed, 137 insertions(+), 21 deletions(-) diff --git a/backend/audit/templates/audit/manage-submission-change-access.html b/backend/audit/templates/audit/manage-submission-change-access.html index f377161673..5b4476d242 100644 --- a/backend/audit/templates/audit/manage-submission-change-access.html +++ b/backend/audit/templates/audit/manage-submission-change-access.html @@ -8,16 +8,24 @@ method="post"> {% csrf_token %}
-

Change {{ friendly_role }}

-

- There may only be one {{ friendly_role }} per single audit submission. By submitting a new official, you will be removing the current official listed below. -

-

- Name: {{ certifier_name }} -

-

- Email: {{ email }} -

+ + {% if role != "editor" %} +

Change {{ friendly_role }}

+

+ There may only be one {{ friendly_role }} per single audit submission. By submitting a new official, you will be removing the current official listed below. +

+

+ Name: {{ certifier_name }} +

+

+ Email: {{ email }} +

+ {% else %} +

Add editor

+

+ Grant basic access to this submission to a new user by entering their name and email address below: +

+ {% endif %}
{% if errors %}
    diff --git a/backend/audit/test_manage_submission_access_view.py b/backend/audit/test_manage_submission_access_view.py index a6a1dd7e50..d3f3400529 100644 --- a/backend/audit/test_manage_submission_access_view.py +++ b/backend/audit/test_manage_submission_access_view.py @@ -25,6 +25,94 @@ def _make_user_and_sac(**kwargs): return user, sac +class ChangeOrAddRoleViewTests(TestCase): + """ + GET and POST tests for adding editors to a submission. + """ + + role = "editor" + view = "audit:ChangeOrAddRoleView" + + def test_basic_get(self): + """ + A user should be able to access this page for a SAC they're associated with. + """ + user, sac = _make_user_and_sac() + baker.make(Access, user=user, sac=sac, role="editor") + sac.general_information = {"auditee_uei": "YESIAMAREALUEI"} + sac.save() + + self.client.force_login(user) + url = reverse(self.view, kwargs={"report_id": sac.report_id}) + response = self.client.get(url) + + self.assertEqual(response.status_code, 200) + self.assertIn("YESIAMAREALUEI", response.content.decode("UTF-8")) + + def test_basic_post(self): + """ + Submitting the form with a new email address should create a new Access. + """ + user = baker.make(User, email="adding_user@example.com") + sac = baker.make(SingleAuditChecklist) + baker.make(Access, user=user, sac=sac, role="editor") + sac.general_information = {"auditee_uei": "YESIAMAREALUEI"} + sac.save() + self.client.force_login(user) + + data = { + "fullname": "The New Editor", + "email": "neweditoruser@example.gov", + } + + url = reverse(self.view, kwargs={"report_id": sac.report_id}) + response = self.client.post(url, data=data) + self.assertEqual(302, response.status_code) + + newaccess = Access.objects.get( + sac=sac, fullname=data["fullname"], email=data["email"] + ) + self.assertEqual(self.role, newaccess.role) + + def test_login_required(self): + """When an unauthenticated request is made""" + + response = self.client.get( + reverse( + self.view, + kwargs={"report_id": "12345"}, + ) + ) + + self.assertEqual(response.status_code, 403) + + def test_bad_report_id_returns_403(self): + """ + When a request is made for a malformed or nonexistent report_id, + a 403 error should be returned + """ + user = baker.make(User) + + self.client.force_login(user) + + response = self.client.get( + reverse(self.view, kwargs={"report_id": "this is not a report id"}) + ) + + self.assertEqual(response.status_code, 403) + + def test_inaccessible_audit_returns_403(self): + """When a request is made for an audit that is inaccessible for this user, a 403 error should be returned""" + user, sac = _make_user_and_sac() + + self.client.force_login(user) + response = self.client.post( + reverse(self.view, kwargs={"report_id": sac.report_id}) + ) + + self.assertEqual(response.status_code, 403) + + class ChangeAuditorCertifyingOfficialViewTests(TestCase): """ GET and POST tests for changing auditor certifying official. diff --git a/backend/audit/urls.py b/backend/audit/urls.py index f4b6404b2d..220bfd96a5 100644 --- a/backend/audit/urls.py +++ b/backend/audit/urls.py @@ -100,6 +100,11 @@ def camel_to_hyphen(raw: str) -> str: views.ChangeAuditeeCertifyingOfficialView.as_view(), name="ChangeAuditeeCertifyingOfficial", ), + path( + "manage-submission/add-editor/", + views.ChangeOrAddRoleView.as_view(), + name="ChangeOrAddRoleView", + ), ] for form_section in FORM_SECTIONS: diff --git a/backend/audit/views/__init__.py b/backend/audit/views/__init__.py index bd1a9954d8..6b08bc5636 100644 --- a/backend/audit/views/__init__.py +++ b/backend/audit/views/__init__.py @@ -1,6 +1,7 @@ from .home import Home from .manage_submission import ManageSubmissionView from .manage_submission_access import ( + ChangeOrAddRoleView, ChangeAuditeeCertifyingOfficialView, ChangeAuditorCertifyingOfficialView, ) @@ -36,6 +37,7 @@ AuditorCertificationStep1View, AuditorCertificationStep2View, CertificationView, + ChangeOrAddRoleView, ChangeAuditeeCertifyingOfficialView, ChangeAuditorCertifyingOfficialView, CrossValidationView, diff --git a/backend/audit/views/manage_submission_access.py b/backend/audit/views/manage_submission_access.py index a20641bc64..66f48af016 100644 --- a/backend/audit/views/manage_submission_access.py +++ b/backend/audit/views/manage_submission_access.py @@ -29,15 +29,13 @@ class ChangeAccessForm(forms.Form): # ) -class ChangeAuditorCertifyingOfficialView( - SingleAuditChecklistAccessRequiredMixin, generic.View -): +class ChangeOrAddRoleView(SingleAuditChecklistAccessRequiredMixin, generic.View): """ - View for changing the auditor certifying official + View for adding a new editor; also has logic for changing certifying roles. """ - role = "certifying_auditor_contact" - other_role = "certifying_auditee_contact" + role = "editor" + other_role = "" def get(self, request, *args, **kwargs): """ @@ -45,17 +43,23 @@ def get(self, request, *args, **kwargs): """ report_id = kwargs["report_id"] sac = SingleAuditChecklist.objects.get(report_id=report_id) - access = Access.objects.get(sac=sac, role=self.role) context = { "role": self.role, - "friendly_role": access.get_friendly_role(), + "friendly_role": None, "auditee_uei": sac.general_information["auditee_uei"], "auditee_name": sac.general_information.get("auditee_name"), - "certifier_name": access.fullname, - "email": access.email, + "certifier_name": None, + "email": None, "report_id": report_id, "errors": [], } + if self.role != "editor": + access = Access.objects.get(sac=sac, role=self.role) + context = context | { + "friendly_role": access.get_friendly_role(), + "certifier_name": access.fullname, + "email": access.email, + } return render(request, "audit/manage-submission-change-access.html", context) @@ -114,7 +118,16 @@ def post(self, request, *args, **kwargs): return redirect(url) -class ChangeAuditeeCertifyingOfficialView(ChangeAuditorCertifyingOfficialView): +class ChangeAuditorCertifyingOfficialView(ChangeOrAddRoleView): + """ + View for changing the auditor certifying official + """ + + role = "certifying_auditor_contact" + other_role = "certifying_auditee_contact" + + +class ChangeAuditeeCertifyingOfficialView(ChangeOrAddRoleView): """ View for changing the auditee certifying official """ From 980e1d6c9e1ef50e8a00c80bcac0aaabcc47cbfb Mon Sep 17 00:00:00 2001 From: Tadhg O'Higgins <2626258+tadhg-ohiggins@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:29:24 -0800 Subject: [PATCH 4/4] Use actual friendly status for comparison. (#2997) --- backend/audit/templates/audit/my_submissions.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/audit/templates/audit/my_submissions.html b/backend/audit/templates/audit/my_submissions.html index f24f186718..6bc3f86605 100644 --- a/backend/audit/templates/audit/my_submissions.html +++ b/backend/audit/templates/audit/my_submissions.html @@ -34,7 +34,7 @@

    Audits in progress

    {{ item.submission_status }} - {% if item.submission_status == "Ready For Certification" %} + {% if item.submission_status == "Ready for Certification" %}