Merge pull request #177 from QGEP/datamodel_delta_1_5_6_vsa_kek_updat…

…e_2019

Create delta_1.5.6_kek_model_update_2008_2019.sql

.deploy/create-release.py → .deploy/create-dumps.py

@@ -31,6 +31,18 @@
 import subprocess
 
 
+def _cmd(args):
+    """
+    Runs a command in subprocess, showing output if it fails
+    """
+
+    try:
+        subprocess.check_output(args, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        print(e.output)
+        raise e
+
+
 def files_description(version):
     return """
 
@@ -72,7 +84,7 @@ def create_plain_structure_only():
 
     print('Creating dump {}'.format(dump_s))
     dump_file_s = 'artifacts/{dump}'.format(dump=dump_s)
-    subprocess.call(['pg_dump',
+    _cmd(['pg_dump',
                      '--format', 'plain',
                      '--schema-only',
                      '--file', dump_file_s,
@@ -85,7 +97,7 @@ def create_plain_structure_only():
         version=os.environ['CI_TAG'])
     print('Creating dump {}'.format(dump_i))
     dump_file_i = 'artifacts/{dump}'.format(dump=dump_i)
-    subprocess.call(['pg_dump',
+    _cmd(['pg_dump',
                      '--format', 'plain',
                      '--data-only',
                      '--file', dump_file_i,
@@ -118,7 +130,7 @@ def create_plain_value_list(structure_dump_file):
     print('Creating dump {}'.format(dump))
     dump_file = 'artifacts/{dump}'.format(dump=dump)
 
-    subprocess.call(['pg_dump',
+    _cmd(['pg_dump',
                      '--format', 'plain',
                      '--blobs',
                      '--data-only',
@@ -153,7 +165,7 @@ def create_backup_data():
     print('::group::{}'.format(dump))
     print('Creating dump {}'.format(dump))
     dump_file = 'artifacts/{dump}'.format(dump=dump)
-    subprocess.call(['pg_dump',
+    _cmd(['pg_dump',
                      '--format', 'custom',
                      '--blobs',
                      '--data-only',
@@ -178,7 +190,7 @@ def create_backup_complete():
     print('::group::{}'.format(dump))
     print('Creating dump {}'.format(dump))
     dump_file = 'artifacts/{dump}'.format(dump=dump)
-    subprocess.call(['pg_dump',
+    _cmd(['pg_dump',
                      '--format', 'custom',
                      '--blobs',
                      '--compress', '5',
@@ -193,9 +205,7 @@ def create_backup_complete():
 
 def main():
     """
-    Publish the files in a release on github
-    If a release already exist, it will copy its data (title, description, etc),
-    delete it and create a new one with the same data and adding the dump files
+    Creates dumps to be attached to releases.
     """
     if 'CI_TAG' not in os.environ or not os.environ['CI_TAG']:
         print('No git tag: not deploying anything')

.github/workflows/docker-test-and-push.yaml

@@ -41,18 +41,11 @@ jobs:
         run: docker build -f .docker/Dockerfile --build-arg POSTGIS_VERSION=${{ matrix.pgis }} --tag opengisch/qgep_datamodel:${GITHUB_REF##*/}-${{ matrix.pgis }} .
 
       - name: "initialize qgep container"
-        run: |
-          docker run -d -p 5432:5432 --name qgep opengisch/qgep_datamodel:${GITHUB_REF##*/}-${{ matrix.pgis }}
-          docker exec qgep init_qgep.sh wait
+        run: docker run -d -p 5432:5432 --name qgep opengisch/qgep_datamodel:${GITHUB_REF##*/}-${{ matrix.pgis }}
 
-      - name: "create release"
-        run: |
-          CI_TAG=${GITHUB_REF#refs/*/} .deploy/create-release.py
-          ls artifacts
-        env:
-          PGHOST: localhost
-          PGPASSWORD: postgres
-          PGUSER: postgres
+      - name: "wait for qgep container"
+        run: docker exec qgep init_qgep.sh wait
+        timeout-minutes: 10
 
       - name: "run tests on qgep_build (structure)"
         run: |
@@ -87,19 +80,31 @@ jobs:
           docker tag opengisch/qgep_datamodel:${GITHUB_REF##*/}-${{ matrix.pgis }} opengisch/qgep_datamodel:latest
           docker push opengisch/qgep_datamodel:latest
 
+      - name: "create dumps"
+        if: github.event_name != 'pull_request' && matrix.pgis == env.pgis_stable
+        run: |
+          CI_TAG=${GITHUB_REF#refs/*/} .deploy/create-dumps.py
+          ls artifacts
+        env:
+          PGHOST: localhost
+          PGPASSWORD: postgres
+          PGUSER: postgres
+
       - uses: AButler/[email protected]
         if: github.event_name == 'release' && matrix.pgis == env.pgis_stable
         with:
           files: 'artifacts/*'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Setup tmate session
-        if: failure()
-        uses: mxschmitt/action-tmate@v3
-        with:
-          limit-access-to-actor: true
-
       - name: "failure logs"
         if: failure()
         run: |
           docker logs qgep
+
+      # TMATE disabled for now as it hangs process, we should do this instead :
+      # https://github.com/marketplace/actions/debugging-with-tmate#manually-triggered-debug
+      # - name: Setup tmate session
+      #   if: failure()
+      #   uses: mxschmitt/action-tmate@v3
+      #   with:
+      #     limit-access-to-actor: true

03_qgep_db_dss.sql

@@ -1055,7 +1055,7 @@ ALTER TABLE qgep_od.maintenance_event ADD COLUMN data_details  varchar(50) ;
 COMMENT ON COLUMN qgep_od.maintenance_event.data_details IS 'yyy_Ort, wo sich weitere Detailinformationen zum Ereignis finden (z.B. Nr. eines Videobandes) / Ort, wo sich weitere Detailinformationen zum Ereignis finden (z.B. Nr. eines Videobandes) / Lieu où se trouvent les données détaillées (par ex. n° d''une bande vidéo)';
 ALTER TABLE qgep_od.maintenance_event ADD COLUMN duration  smallint ;
 COMMENT ON COLUMN qgep_od.maintenance_event.duration IS 'Duration of event in days / Dauer des Ereignisses in Tagen / Durée de l''événement en jours';
-ALTER TABLE qgep_od.maintenance_event ADD COLUMN identifier  varchar(20) ;
+ALTER TABLE qgep_od.maintenance_event ADD COLUMN identifier  varchar(41) ;
 COMMENT ON COLUMN qgep_od.maintenance_event.identifier IS '';
 ALTER TABLE qgep_od.maintenance_event ADD COLUMN kind  integer ;
 COMMENT ON COLUMN qgep_od.maintenance_event.kind IS 'Type of event / Art des Ereignisses / Genre d''événement';
@@ -1065,7 +1065,7 @@ ALTER TABLE qgep_od.maintenance_event ADD COLUMN reason  varchar(50) ;
 COMMENT ON COLUMN qgep_od.maintenance_event.reason IS 'Reason for this event / Ursache für das Ereignis / Cause de l''événement';
 ALTER TABLE qgep_od.maintenance_event ADD COLUMN remark  varchar(80) ;
 COMMENT ON COLUMN qgep_od.maintenance_event.remark IS 'General remarks / Allgemeine Bemerkungen / Remarques générales';
-ALTER TABLE qgep_od.maintenance_event ADD COLUMN result  varchar(50) ;
+ALTER TABLE qgep_od.maintenance_event ADD COLUMN result  varchar(255) ;
 COMMENT ON COLUMN qgep_od.maintenance_event.result IS 'Result or important comments for this event / Resultat oder wichtige Bemerkungen aus Sicht des Bearbeiters / Résultat ou commentaire importante de l''événement';
 ALTER TABLE qgep_od.maintenance_event ADD COLUMN status  integer ;
 COMMENT ON COLUMN qgep_od.maintenance_event.status IS 'Disposition state of the maintenance event / Phase in der sich das Erhaltungsereignis befindet / Phase dans laquelle se trouve l''événement de maintenance';
@@ -4226,7 +4226,7 @@ ALTER TABLE qgep_vl.reach_point_outlet_shape ADD CONSTRAINT pkey_qgep_vl_reach_p
  REFERENCES qgep_vl.reach_point_outlet_shape (code) MATCH SIMPLE 
  ON UPDATE RESTRICT ON DELETE RESTRICT;
 ALTER TABLE qgep_od.reach_point ADD COLUMN fk_wastewater_networkelement varchar (16);
-ALTER TABLE qgep_od.reach_point ADD CONSTRAINT rel_reach_point_wastewater_networkelement FOREIGN KEY (fk_wastewater_networkelement) REFERENCES qgep_od.wastewater_networkelement(obj_id) ON UPDATE CASCADE ON DELETE set null;
+ALTER TABLE qgep_od.reach_point ADD CONSTRAINT rel_reach_point_wastewater_networkelement FOREIGN KEY (fk_wastewater_networkelement) REFERENCES qgep_od.wastewater_networkelement(obj_id) ON UPDATE CASCADE ON DELETE set null DEFERRABLE INITIALLY IMMEDIATE;
 ALTER TABLE qgep_od.wastewater_node ADD CONSTRAINT oorel_od_wastewater_node_wastewater_networkelement FOREIGN KEY (obj_id) REFERENCES qgep_od.wastewater_networkelement(obj_id) ON DELETE CASCADE ON UPDATE CASCADE;
 ALTER TABLE qgep_od.wastewater_node ADD COLUMN fk_hydr_geometry varchar (16);
 ALTER TABLE qgep_od.wastewater_node ADD CONSTRAINT rel_wastewater_node_hydr_geometry FOREIGN KEY (fk_hydr_geometry) REFERENCES qgep_od.hydr_geometry(obj_id) ON UPDATE CASCADE ON DELETE cascade;
@@ -4395,7 +4395,7 @@ ALTER TABLE qgep_vl.structure_part_renovation_demand ADD CONSTRAINT pkey_qgep_vl
  REFERENCES qgep_vl.structure_part_renovation_demand (code) MATCH SIMPLE 
  ON UPDATE RESTRICT ON DELETE RESTRICT;
 ALTER TABLE qgep_od.structure_part ADD COLUMN fk_wastewater_structure varchar (16);
-ALTER TABLE qgep_od.structure_part ADD CONSTRAINT rel_structure_part_wastewater_structure FOREIGN KEY (fk_wastewater_structure) REFERENCES qgep_od.wastewater_structure(obj_id) ON UPDATE CASCADE ON DELETE cascade;
+ALTER TABLE qgep_od.structure_part ADD CONSTRAINT rel_structure_part_wastewater_structure FOREIGN KEY (fk_wastewater_structure) REFERENCES qgep_od.wastewater_structure(obj_id) ON UPDATE CASCADE ON DELETE cascade DEFERRABLE INITIALLY IMMEDIATE;
 ALTER TABLE qgep_od.dryweather_downspout ADD CONSTRAINT oorel_od_dryweather_downspout_structure_part FOREIGN KEY (obj_id) REFERENCES qgep_od.structure_part(obj_id) ON DELETE CASCADE ON UPDATE CASCADE;
 ALTER TABLE qgep_od.access_aid ADD CONSTRAINT oorel_od_access_aid_structure_part FOREIGN KEY (obj_id) REFERENCES qgep_od.structure_part(obj_id) ON DELETE CASCADE ON UPDATE CASCADE;
 CREATE TABLE qgep_vl.access_aid_kind () INHERITS (qgep_sys.value_list_base);