Better summary, show all bad urls at the end

.cicd/tests.py

@@ -89,6 +89,12 @@ def process_each(softwares):
             # unreachable urls... they may be pointing to salt:// for example
             if version.get("skip_urltest", False):
                 count_status["skipped"] += 1
+                msg = "SKIPPED : %s -- %s" % (s, v)
+                print(msg)
+                continue
+            if not version.get("installer", ""):
+                msg = "SKIPPED (No URL) : %s -- %s" % (s, v)
+                print(msg)
                 continue
             # Testing each non-salt URL for availability
             scheme = urlparse(version.get("installer", "")).scheme
@@ -202,49 +208,65 @@ def process_each(softwares):
 label_rev = {"AMD64": "x86", "x86": "AMD64"}
 
 for file in our_files:
-#    try:
-    label = " ( {} ) ".format(file)
-    count = 80 - len(label)
-    count = count - count % 2
-    count = round( count / 2)
-    label = ">" * count + label + "<" * count
-    print(label + "<" * (80 - len(label)))
-    with open(file, "r") as stream:
-        template = stream.read()
-    if "cpuarch" in template:
-        for cpuarch in ["AMD64", "x86"]:
-            label = " ( arch: {} ) ".format(label_rev[cpuarch])
-            count = 80 - len(label)
-            count = count - count % 2
-            count = round( count / 2)
-            label = "-" * count + label + "-" * count
-            print(label + "-" * (80 - len(label)))
+    try:
+        label = " ( {} ) ".format(file)
+        count = 80 - len(label)
+        count = count - count % 2
+        count = round( count / 2)
+        label = ">" * count + label + "<" * count
+        print(label + "<" * (80 - len(label)))
+        with open(file, "r") as stream:
+            template = stream.read()
+        if "cpuarch" in template:
+            for cpuarch in ["AMD64", "x86"]:
+                label = " ( arch: {} ) ".format(label_rev[cpuarch])
+                count = 80 - len(label)
+                count = count - count % 2
+                count = round( count / 2)
+                label = "-" * count + label + "-" * count
+                print(label + "-" * (80 - len(label)))
+                caller = salt.client.Caller(".cicd/minion")
+                caller.cmd("grains.set", "cpuarch", cpuarch)
+                data = caller.cmd("winrepo.show_sls", file)
+                process_each(data)
+        else:
             caller = salt.client.Caller(".cicd/minion")
-            caller.cmd("grains.set", "cpuarch", cpuarch)
             data = caller.cmd("winrepo.show_sls", file)
             process_each(data)
-    else:
-        caller = salt.client.Caller(".cicd/minion")
-        data = caller.cmd("winrepo.show_sls", file)
-        process_each(data)
-#    except Exception:
-#        exc = sys.exc_info()[0]
-#        print("[EXCEPTION] " + str(exc))
-#        traceback.print_exc()
-#        count_status["exceptions"] += 1
-#        pass
+    except Exception:
+        exc = sys.exc_info()[0]
+        print("[EXCEPTION] " + str(exc))
+        traceback.print_exc()
+        count_status["exceptions"] += 1
+        pass
+
 print("-" * 80 + "\n")
 
+if not TEST_STATUS:
+    print("URLs With Failures:")
+    print("*" * 80)
+    print("\n".join(TEST_FAILURES))
+    print("*" * 80)
+
+print("")
+print("Content Type:")
+print("=============")
 print(tabulate(count_c_types.most_common(), ["Total", sum(count_c_types.values())]) + "\n")
+print("")
+print("HTTP Codes:")
+print("===========")
 print(tabulate(count_http_codes.most_common(), ["Total", sum(count_http_codes.values())]) + "\n")
+print("")
+print("Test Summary:")
+print("=============")
 print(tabulate(count_status.most_common(), ["Total", sum(count_status.values())]) + "\n")
 
 if not TEST_STATUS:
-    print("*" * 80)
-    print("\n".join(TEST_FAILURES))
     print("*" * 80)
     print("BUILD FAILING. You can grep for 'PROBLEM HERE' to find out how to fix this.")
     print("*" * 80)
     exit(1)
 
+print("*" * 80)
 print("Everything went smoothly. No errors were found. Happy deployment!")
+print("*" * 80)

7zip.sls

@@ -24,25 +24,26 @@
     reboot: False
 {% endfor %}
 {% endfor %}
-#
-#
+
+
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# Below are versions of 7-zip that have had the installer assets removed from the web.
-# An uninstall only definition will remain here so the packages will show up
-# correctly in `pkg.list_pkgs` and to allow for removal using `pkg.remove`
+# Below are versions of 7-zip that have had the installer assets removed from
+# the normal website. The binaries are still available on sourceforge.
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+{% set source_path = 'https://downloads.sourceforge.net/project/sevenzip/7-Zip/' %}
+
 {% set versions = { '18':['06', '05', '03', '01'], '16':['04', '03', '02', '00']} %}
 {% for major, subversions in versions.items() %}
 {% for minor in subversions %}
   '{{major}}.{{minor}}.00.0':
     {% if grains['cpuarch'] == 'AMD64' %}
     full_name: '7-Zip {{major}}.{{minor}} (x64 edition)'
-    installer: '{{ source_path }}7z{{major}}{{minor}}-x64.msi'
+    installer: '{{ source_path }}{{ major }}.{{ minor }}/7z{{major}}{{minor}}-x64.msi'
     uninstaller: '{{ source_path }}7z{{major}}{{minor}}-x64.msi'
     arch: x64
     {% else %}
     full_name: '7-Zip {{major}}.{{minor}}'
-    installer: '{{ source_path }}7z{{major}}{{minor}}.msi'
+    installer: '{{ source_path }}{{ major }}.{{ minor }}/7z{{major}}{{minor}}.msi'
     uninstaller: '{{ source_path }}7z{{major}}{{minor}}.msi'
     arch: x86
     {% endif %}

adobeair.sls

@@ -1,18 +1,9 @@
 adobeair:
   latest:
     full_name: 'Adobe AIR'
-    installer: 'http://airdownload.adobe.com/air/win/download/30.0/AdobeAIRInstaller.exe'
+    installer: https://airsdk.harman.com/assets/downloads/AdobeAIR.exe
     install_flags: '-silent -eulaAccepted'
-    uninstaller: 'c:\salt\var\cache\salt\minion\extrn_files\base\airdownload.adobe.com\air\win\download\30.0\AdobeAIRInstaller.exe'
-    uninstall_flags: '-uninstall'
-    msiexec: False
-    locale: en_US
-    reboot: False
-  '30.0.0.107':
-    full_name: 'Adobe AIR'
-    installer: 'http://airdownload.adobe.com/air/win/download/30.0/AdobeAIRInstaller.exe'
-    install_flags: '-silent -eulaAccepted'
-    uninstaller: 'c:\salt\var\cache\salt\minion\extrn_files\base\airdownload.adobe.com\air\win\download\30.0\AdobeAIRInstaller.exe'
+    uninstaller: 'c:\salt\var\cache\salt\minion\extrn_files\base\airsdk.harmon.com\assets\downloads\AdobeAIR.exe'
     uninstall_flags: '-uninstall'
     msiexec: False
     locale: en_US

audacity.sls

@@ -1,34 +1,77 @@
-{% set versions = ['3.0.0'] %}
+{% set versions = [
+    '3.2.1',
+    '3.2.0',
+    '3.1.3',
+    '3.1.2',
+    '3.1.1',
+    '3.1.0',
+    '3.0.5',
+    '3.0.3',
+] %}
 {% if grains['cpuarch'] == 'AMD64' %}
     {% set PROGRAM_FILES = "%ProgramFiles(x86)%" %}
+    {% set ARCH = "64bit" %}
 {% else %}
     {% set PROGRAM_FILES = "%ProgramFiles%" %}
+    {% set ARCH = "32bit" %}
 {% endif %}
+{% set source_path = 'https://github.com/audacity/audacity/releases/download/' %}
 audacity:
   {% for version in versions %}
   '{{ version }}':
     full_name: 'Audacity {{ version }}'
-    # official page has counter measures for automaitc downloads
-    #installer: 'https://www.fosshub.com/Audacity.html?dwl=audacity-win-{{ version }}.exe'
-    installer: 'https://fossies.org/windows/misc/audacity-win-{{ version }}.exe'
+    installer: {{ source_path }}Audacity-{{ version }}/audacity-win-{{ version }}-{{ ARCH }}.exe
+    install_flags: '/SP- /verysilent /norestart'
+    uninstaller: '%ProgramFiles%\Audacity\unins000.exe'
+    uninstall_flags: '/SP- /verysilent /norestart'
+    msiexec: False
+    locale: en_US
+    reboot: False
+  {% endfor %}
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# These have a different filename format
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+{% set versions = [
+    '3.0.2',
+    '3.0.0',
+] %}
+  {% for version in versions %}
+  '{{ version }}':
+    full_name: 'Audacity {{ version }}'
+    installer: {{ source_path }}Audacity-{{ version }}/audacity-win-{{ version }}.exe
     install_flags: '/SP- /verysilent /norestart'
     uninstaller: '{{ PROGRAM_FILES }}\Audacity\unins000.exe'
     uninstall_flags: '/SP- /verysilent /norestart'
     msiexec: False
     locale: en_US
     reboot: False
   {% endfor %}
-#
-#
+
+
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # Below are versions of Audacity that have had the installer assets moved to the legacy page.
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-  {% for version in ['2.4.2', '2.4.1', '2.3.3', '2.3.2', '2.3.1', '2.3.0', '2.2.2', '2.2.0-rc1', '2.2.1','2.1.3', '2.1.1', '2.1.0', '2.0.6', '2.0'] %}
+{% set versions = [
+    '2.4.2',
+    '2.4.1',
+    '2.3.3',
+    '2.3.2',
+    '2.3.1',
+    '2.3.0',
+    '2.2.2',
+    '2.2.0-rc1',
+    '2.2.1',
+    '2.1.3',
+    '2.1.1',
+    '2.1.0',
+    '2.0.6',
+    '2.0',
+] %}
+  {% for version in versions %}
   '{{ version }}':
     skip_urltest: True
     full_name: 'Audacity {{ version }}'
-    # official page has counter measures for automaitc downloads - no alternative like fossies archive found as yet
-    #installer: "https://www.fosshub.com/Audacity-old.html?dwl=audacity-win-{{ version }}.exe"
     uninstaller: '{{ PROGRAM_FILES }}\Audacity\unins000.exe'
     uninstall_flags: '/SP- /verysilent /norestart'
     msiexec: False

nsis.sls

@@ -1,5 +1,5 @@
 # Define versions
-{% set versions = {'3': ['03', '02.1', '02', '01', '0'],
+{% set versions = {'3': ['08', '07', '06.1', '06', '05', '04', '03', '02.1', '02', '01', '0'],
                    '3%20Pre-release': ['0b2', '0b1'],
                    '2': ['51', '50', '49', '48', '47', '46']} %}
 
@@ -8,7 +8,7 @@ nsis:
   {% for minor in subversions %}
   '{{major|first}}.{{minor}}':
     full_name: 'Nullsoft Install System'
-    installer: 'https://iweb.dl.sourceforge.net/project/nsis/NSIS%20{{major}}/{{major|first}}.{{minor}}/nsis-{{major|first}}.{{minor}}-setup.exe'
+    installer: 'https://downloads.sourceforge.net/project/nsis/NSIS%20{{major}}/{{major|first}}.{{minor}}/nsis-{{major|first}}.{{minor}}-setup.exe'
     install_flags: '/S'
     {% if grains['cpuarch'] == 'AMD64' %}
     uninstaller: '%PROGRAMFILES(x86)%\NSIS\uninst-nsis.exe'

rubyinstaller_x64.sls

@@ -1,8 +1,14 @@
+{% set base_url = "https://github.com/oneclick/rubyinstaller/releases/download/" %}
+{% set versions = (
+    ('2.2.3-p173', '2.2.3', '22'),
+    ('2.1.7-p400', '2.1.7', '21'),
+    ('2.0.0-p647', '2.0.0-p647', '200'),
+) %}
 rubyinstaller_x64:
-  {% for version, dl_version, dsk_version in (('2.2.3-p173', '2.2.3', '22'), ('2.1.7-p400', '2.1.7', '21'), ('2.0.0-p647', '2.0.0-p647', '200')) %}
+{% for version, dl_version, dsk_version in versions %}
   '{{ version }}':
     full_name: 'Ruby {{ version }}-x64'
-    installer: 'http://dl.bintray.com/oneclick/rubyinstaller/rubyinstaller-{{ dl_version }}-x64.exe'
+    installer: {{ base_url }}ruby-{{ dl_version }}/rubyinstaller-{{ dl_version }}-x64.exe
     install_flags: '/verysilent'
     uninstaller: 'C:\Ruby{{ dsk_version }}-x64\unins000.exe'
     uninstall_flags: '/verysilent'

rubyinstaller_x86.sls

@@ -1,12 +1,19 @@
+{% set base_url = "https://github.com/oneclick/rubyinstaller/releases/download/" %}
+{% set versions = (
+    ('2.2.3-p173', '2.2.3', '22'),
+    ('2.1.7-p400', '2.1.7', '21'),
+    ('2.0.0-p647', '2.0.0-p647', '200'),
+    ('1.9.3-p551', '1.9.3-p551', '193'),
+) %}
 rubyinstaller_x86:
-  {% for version, dl_version, dsk_version in (('2.2.3-p173', '2.2.3', '22'), ('2.1.7-p400', '2.1.7', '21'), ('2.0.0-p647', '2.0.0-p647', '200'), ('1.9.3-p551', '1.9.3-p551', '193')) %}
+{% for version, dl_version, dsk_version in versions %}
   '{{ version }}':
     full_name: 'Ruby {{ version }}'
-    installer: 'http://dl.bintray.com/oneclick/rubyinstaller/rubyinstaller-{{ dl_version }}.exe'
+    installer: '{{ base_url }}ruby-{{ dl_version }}/rubyinstaller-{{ dl_version }}.exe'
     install_flags: '/verysilent'
     uninstaller: 'C:\Ruby{{ dsk_version }}\unins000.exe'
     uninstall_flags: '/verysilent'
     msiexec: False
     locale: en_US
     reboot: False
-  {% endfor %}
+{% endfor %}