diff --git a/Jenkinsfile b/Jenkinsfile index 7a631cc1a..437574bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -81,14 +81,14 @@ spec: } } // Verifies that the -javadoc and -sources artifacts can be generated (by enabling the - // javadoc profile contained in three pom.xml files). Also verifies that the build + // prepare-release profile contained in four pom.xml files). Also verifies that the build // is reproducible, and that Spotbugs checks do not fail (cf. // https://eclipse.github.io/steady/contributor/#contribution-content-guidelines). - stage('Create javadoc + sources, Verify Spotbugs and Reproducibility') { + stage('Create javadoc + sources + CycloneDX BOM, Verify Spotbugs and Reproducibility') { steps { container('maven') { sh 'export MAVEN_OPTS="-Xms4g -Xmx8g"' - sh 'mvn -B -e -P gradle,javadoc \ + sh 'mvn -B -e -P gradle,prepare-release \ -Dspring.standalone \ -DskipTests \ -Dvulas.shared.m2Dir=/home/jenkins/agent/workspace \ @@ -96,7 +96,7 @@ spec: -Dspotbugs.includeFilterFile=findbugs-include.xml \ -Dspotbugs.failOnError=true \ clean install com.github.spotbugs:spotbugs-maven-plugin:4.2.3:check' - // sh 'mvn -B -e -P javadoc \ + // sh 'mvn -B -e -P prepare-release \ // -Dspring.standalone \ // -DskipTests \ // -Dreference.repo=https://repo.maven.apache.org/maven2 \ @@ -132,7 +132,7 @@ spec: sh 'gpg --batch --import "${KEYRING}"' sh 'for fpr in $(gpg --list-keys --with-colons | awk -F: \'/fpr:/ {print $10}\' | sort -u); do echo -e "5\ny\n" | gpg --batch --command-fd 0 --expert --edit-key ${fpr} trust; done' } - sh 'mvn -B -e -P gradle,javadoc,release \ + sh 'mvn -B -e -P gradle,prepare-release,release \ -Dspring.standalone \ -DskipTests \ clean deploy' diff --git a/docker/.env.sample b/docker/.env.sample index c6f025118..ed79b2124 100644 --- a/docker/.env.sample +++ b/docker/.env.sample @@ -34,9 +34,7 @@ https_proxy= no_proxy= # kb-importer update cron expression -KB_IMPORTER_CRON_HOUR=0 KB_IMPORTER_STATEMENTS_FOLDER=statements KB_IMPORTER_STATEMENTS_REPO=https://github.com/sap/project-kb KB_IMPORTER_STATEMENTS_BRANCH=vulnerability-data KB_IMPORTER_SKIP_CLONE=True -KB_IMPORTER_CLONE_FOLDER=repo-clones diff --git a/docker/cache/nginx.conf b/docker/cache/conf/nginx.conf similarity index 100% rename from docker/cache/nginx.conf rename to docker/cache/conf/nginx.conf diff --git a/docker/docker-compose-new.yml b/docker/docker-compose-new.yml deleted file mode 100755 index 77810a8d8..000000000 --- a/docker/docker-compose-new.yml +++ /dev/null @@ -1,181 +0,0 @@ -version: '2.4' - -# This Docker Compose application makes use of profiles, available as of Docker -# Compose 1.28. -# -# - Core services: haproxy, rest-backend and postgresql need to always run -# - UI services: frontend-apps, frontend-bugs and cache deliver OpenUI5 Web -# applications for scan results (http://localhost:8033/apps) and -# vulnerabilities (http://localhost:8033/bugs). -# - VDB services: rest-lib-utils, kb-importer and patch-lib-analyzer update -# the vulnerability database and resolve unassessed findings (by comparing -# method bodies obtained from rest-lib-utils) -# -# The different profiles can be started using start-steady.sh or using Docker -# Compose's --profile option, e.g. docker-compose --profile ui up -d --build - -services: - - # Core services - haproxy: - container_name: steady-haproxy - hostname: haproxy - env_file: .env - image: haproxy:2.3-alpine - ports: - - "8033:8080" - - "8034:7070" - volumes: - - "./conf/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg" - command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] - depends_on: - - rest-backend - security_opt: - - no-new-privileges - restart: always - - rest-backend: - container_name: steady-rest-backend - hostname: rest-backend - env_file: - - .env - - ./conf/rest-backend/restbackend.properties - image: eclipse/steady-rest-backend:${VULAS_RELEASE} - expose: - - "8091" - environment: - - DELAY_STARTUP=5 - - vulas.shared.cia.serviceUrl=http://rest-lib-utils:8092/cia - - vulas.shared.cve.serviceUrl=https://services.nvd.nist.gov/rest/json/cve/1.0/ - - spring.datasource.username=${POSTGRES_USER} - - spring.datasource.password=${POSTGRES_PASSWORD} - volumes: - - "./data/rest-backend:/flyway-callbacks" - depends_on: - - postgresql - security_opt: - - no-new-privileges - restart: always - - postgresql: - container_name: steady-postgresql - hostname: postgresql - image: postgres:11-alpine - environment: - - POSTGRES_DB=vulas - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - PGDATA=/var/lib/postgresql/data - ports: - - "8032:5432" - volumes: - - steady-postgres-data:/var/lib/postgresql/data - - ./conf/postgresql:/docker-entrypoint-initdb.d:ro - security_opt: - - no-new-privileges - restart: always - - # UI services - frontend-apps: - container_name: steady-frontend-apps - hostname: frontend-apps - image: eclipse/steady-frontend-apps:${VULAS_RELEASE} - expose: - - "8080" - depends_on: - - rest-backend - - cache - security_opt: - - no-new-privileges - restart: always - profiles: - - ui - - frontend-bugs: - container_name: steady-frontend-bugs - hostname: frontend-bugs - image: eclipse/steady-frontend-bugs:${VULAS_RELEASE} - expose: - - "8080" - depends_on: - - rest-backend - security_opt: - - no-new-privileges - restart: always - profiles: - - ui - - cache: - container_name: steady-cache - hostname: cache - image: nginx:alpine - expose: - - "80" - volumes: - - ./conf/cache/nginx.conf:/etc/nginx/nginx.conf:ro - - ./data/cache/:/tmp/cache_all/ - security_opt: - - no-new-privileges - restart: always - profiles: - - ui - - # VDB services - patch-lib-analyzer: - container_name: steady-patch-lib-analyzer - hostname: patch-lib-analyzer - image: eclipse/steady-patch-lib-analyzer:${VULAS_RELEASE} - expose: - - "8080" - volumes: - - "./data/patch-lib-analyzer:/patcheval-data" - depends_on: - - rest-backend - - rest-lib-utils - environment: - - PATCHEVAL_OPTS=-bug "" -folder /patcheval-data -j -h 0 -p 6 - - vulas.shared.cia.serviceUrl=http://rest-lib-utils:8092/cia - - vulas.shared.backend.serviceUrl=http://rest-backend:8091/backend - - vulas.patchEval.onlyAddNewResults=true - security_opt: - - no-new-privileges - restart: always - profiles: - - vdb - - rest-lib-utils: - container_name: steady-rest-lib-utils - hostname: rest-lib-utils - image: eclipse/steady-rest-lib-utils:${VULAS_RELEASE} - expose: - - "8092" - volumes: - - "./data/rest-lib-utils:/root/" - security_opt: - - no-new-privileges - restart: always - profiles: - - vdb - - kb-importer: - container_name: steady-kb-importer - image: eclipse/steady-kb-importer:${VULAS_RELEASE} - env_file: .env - volumes: - - "./conf/kb-importer:/kb-importer/conf" - - "./certs:/kb-importer/certs" - - "./data/kb-importer:/kb-importer/data:delegated" - environment: - - CIA_SERVICE_URL=http://rest-lib-utils:8092/cia - - BACKEND_SERVICE_URL=http://rest-backend:8091/backend - depends_on: - - rest-backend - - rest-lib-utils - security_opt: - - no-new-privileges - restart: always - profiles: - - vdb - -volumes: - steady-postgres-data: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8296aa874..6e024b221 100755 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,26 +1,22 @@ version: '2.4' -services: - frontend-apps: - container_name: steady-frontend-apps - hostname: frontend-apps - image: eclipse/steady-frontend-apps:${VULAS_RELEASE} - expose: - - "8080" - security_opt: - - no-new-privileges - restart: always +# This Docker Compose application makes use of profiles, available as of Docker +# Compose 1.28. +# +# - Core services: haproxy, rest-backend and postgresql need to always run +# - UI services: frontend-apps, frontend-bugs and cache deliver OpenUI5 Web +# applications for scan results (http://localhost:8033/apps) and +# vulnerabilities (http://localhost:8033/bugs). +# - VDB services: rest-lib-utils, kb-importer and patch-lib-analyzer update +# the vulnerability database and resolve unassessed findings (by comparing +# method bodies obtained from rest-lib-utils) +# +# The different profiles can be started using start-steady.sh or using Docker +# Compose's --profile option, e.g. docker-compose --profile ui up -d --build - frontend-bugs: - container_name: steady-frontend-bugs - hostname: frontend-bugs - image: eclipse/steady-frontend-bugs:${VULAS_RELEASE} - expose: - - "8080" - security_opt: - - no-new-privileges - restart: always +services: + # Core services haproxy: container_name: steady-haproxy hostname: haproxy @@ -33,32 +29,30 @@ services: - "./haproxy/conf/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg" command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] depends_on: - - frontend-apps - - frontend-bugs - rest-backend - - rest-lib-utils security_opt: - no-new-privileges restart: always - patch-lib-analyzer: - container_name: steady-patch-lib-analyzer - hostname: patch-lib-analyzer - image: eclipse/steady-patch-lib-analyzer:${VULAS_RELEASE} + rest-backend: + container_name: steady-rest-backend + hostname: rest-backend + env_file: + - .env + - ./rest-backend/conf/restbackend.properties + image: eclipse/steady-rest-backend:${VULAS_RELEASE} expose: - - "8080" + - "8091" + environment: + - DELAY_STARTUP=5 + - vulas.shared.cia.serviceUrl=http://rest-lib-utils:8092/cia + - vulas.shared.cve.serviceUrl=https://services.nvd.nist.gov/rest/json/cve/1.0/ + - spring.datasource.username=${POSTGRES_USER} + - spring.datasource.password=${POSTGRES_PASSWORD} volumes: - - "./data/patcheval:/patcheval-data" - links: - - rest-backend:backend - - rest-lib-utils:cia + - "./rest-backend/data:/flyway-callbacks" depends_on: - - rest-backend - environment: - - PATCHEVAL_OPTS=-bug "" -folder /patcheval-data -j -h 0 -p 6 - - vulas.shared.cia.serviceUrl=http://cia:8092/cia - - vulas.shared.backend.serviceUrl=http://backend:8091/backend - - vulas.patchEval.onlyAddNewResults=true + - postgresql security_opt: - no-new-privileges restart: always @@ -76,48 +70,39 @@ services: - "8032:5432" volumes: - steady-postgres-data:/var/lib/postgresql/data - - ./postgresql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:ro + - ./postgresql/conf:/docker-entrypoint-initdb.d:ro security_opt: - no-new-privileges restart: always - rest-backend: - container_name: steady-rest-backend - hostname: rest-backend - env_file: - - .env - - ./rest-backend/conf/restbackend.properties - image: eclipse/steady-rest-backend:${VULAS_RELEASE} + # UI services + frontend-apps: + container_name: steady-frontend-apps + hostname: frontend-apps + image: eclipse/steady-frontend-apps:${VULAS_RELEASE} expose: - - "8091" - environment: - - DELAY_STARTUP=5 - - vulas.shared.cia.serviceUrl=http://cia:8092/cia - - vulas.shared.cve.serviceUrl=https://services.nvd.nist.gov/rest/json/cve/1.0/ - - spring.datasource.username=${POSTGRES_USER} - - spring.datasource.password=${POSTGRES_PASSWORD} - links: - - postgresql:postgresql - - rest-lib-utils:cia - volumes: - - "./data/db-dump/flyway-callbacks:/flyway-callbacks" + - "8080" depends_on: - - postgresql + - rest-backend security_opt: - no-new-privileges restart: always + profiles: + - ui - rest-lib-utils: - container_name: steady-rest-lib-utils - hostname: rest-lib-utils - image: eclipse/steady-rest-lib-utils:${VULAS_RELEASE} + frontend-bugs: + container_name: steady-frontend-bugs + hostname: frontend-bugs + image: eclipse/steady-frontend-bugs:${VULAS_RELEASE} expose: - - "8092" - volumes: - - "./data/rest-lib-utils:/root/" + - "8080" + depends_on: + - rest-backend security_opt: - no-new-privileges restart: always + profiles: + - ui cache: container_name: steady-cache @@ -126,31 +111,72 @@ services: expose: - "80" volumes: - - ./cache/nginx.conf:/etc/nginx/nginx.conf:ro - - ./data/cache/:/tmp/cache_all/ + - ./cache/conf/nginx.conf:/etc/nginx/nginx.conf:ro + - ./cache/data/:/tmp/cache_all/ + security_opt: + - no-new-privileges + restart: always + profiles: + - ui + + # VDB services + patch-lib-analyzer: + container_name: steady-patch-lib-analyzer + hostname: patch-lib-analyzer + image: eclipse/steady-patch-lib-analyzer:${VULAS_RELEASE} + expose: + - "8080" + volumes: + - "./patch-lib-analyzer/data:/patcheval-data" + depends_on: + - rest-backend + - rest-lib-utils + environment: + - PATCHEVAL_OPTS=-bug "" -folder /patcheval-data -j -h 0 -p 6 + - vulas.shared.cia.serviceUrl=http://rest-lib-utils:8092/cia + - vulas.shared.backend.serviceUrl=http://rest-backend:8091/backend + - vulas.patchEval.onlyAddNewResults=true security_opt: - no-new-privileges restart: always + profiles: + - vdb + + rest-lib-utils: + container_name: steady-rest-lib-utils + hostname: rest-lib-utils + image: eclipse/steady-rest-lib-utils:${VULAS_RELEASE} + expose: + - "8092" + volumes: + - "./rest-lib-utils/data:/root/" + security_opt: + - no-new-privileges + restart: always + profiles: + - vdb kb-importer: container_name: steady-kb-importer image: eclipse/steady-kb-importer:${VULAS_RELEASE} + expose: + - "8080" env_file: .env volumes: - "./kb-importer/conf:/kb-importer/conf" - - "./kb-importer/certs:/kb-importer/certs" + - "./certs:/kb-importer/certs" - "./kb-importer/data:/kb-importer/data:delegated" environment: - - CIA_SERVICE_URL=http://cia:8092/cia - - BACKEND_SERVICE_URL=http://backend:8091/backend + - CIA_SERVICE_URL=http://rest-lib-utils:8092/cia + - BACKEND_SERVICE_URL=http://rest-backend:8091/backend depends_on: - rest-backend - links: - - rest-backend:backend - - rest-lib-utils:cia + - rest-lib-utils security_opt: - no-new-privileges restart: always + profiles: + - vdb volumes: steady-postgres-data: diff --git a/docker/frontend-apps/Dockerfile b/docker/frontend-apps/Dockerfile index 453fb1eaf..af9bdc8a5 100644 --- a/docker/frontend-apps/Dockerfile +++ b/docker/frontend-apps/Dockerfile @@ -1,4 +1,5 @@ -FROM jetty:alpine +# https://hub.docker.com/layers/jetty/library/jetty/9.4.46-jdk11-alpine-eclipse-temurin/images/sha256-dcaab143043b8916675f8533700c07310986e94db9ee4f2f6bf336e8befc53e9?context=explore +FROM jetty@sha256:dcaab143043b8916675f8533700c07310986e94db9ee4f2f6bf336e8befc53e9 LABEL maintainer="steady-dev@eclipse.org" diff --git a/docker/frontend-bugs/Dockerfile b/docker/frontend-bugs/Dockerfile index c99ab5f8d..90812fe0f 100644 --- a/docker/frontend-bugs/Dockerfile +++ b/docker/frontend-bugs/Dockerfile @@ -1,4 +1,5 @@ -FROM jetty:alpine +# https://hub.docker.com/layers/jetty/library/jetty/9.4.46-jdk11-alpine-eclipse-temurin/images/sha256-dcaab143043b8916675f8533700c07310986e94db9ee4f2f6bf336e8befc53e9?context=explore +FROM jetty@sha256:dcaab143043b8916675f8533700c07310986e94db9ee4f2f6bf336e8befc53e9 LABEL maintainer="steady-dev@eclipse.org" diff --git a/docker/haproxy/conf/haproxy.cfg b/docker/haproxy/conf/haproxy.cfg index 014c282a1..d2ad98454 100644 --- a/docker/haproxy/conf/haproxy.cfg +++ b/docker/haproxy/conf/haproxy.cfg @@ -40,6 +40,7 @@ frontend http-in acl is_backend path_beg -i /backend acl is_cia path_beg -i /cia + acl is_kb_importer path_beg -i /kb-importer acl is_bugs_admin path_beg -i /bugs acl is_frontend path_beg -i /apps acl to_be_cached_long_term urlp_reg(lastChange) ^[0-9]{13}$ @@ -48,6 +49,7 @@ frontend http-in use_backend cache-long-term-nodes if to_be_cached_long_term cache_long_term_available use_backend rest-backend-nodes if is_backend use_backend rest-lib-utils-nodes if is_cia + use_backend kb-importer-nodes if is_kb_importer use_backend frontend-bugs-nodes if is_bugs_admin use_backend frontend-apps-nodes if is_frontend @@ -72,6 +74,17 @@ backend rest-lib-utils-nodes stats enable server rest-lib-utils rest-lib-utils:8092 check resolvers docker_resolver resolve-prefer ipv4 +backend kb-importer-nodes + mode http + log global + balance roundrobin + option forwardfor + http-request set-header X-Forwarded-Port %[dst_port] + option httpchk GET /kb-importer/ HTTP/1.1\r\nHost:\ haproxy01 + http-check expect ! rstatus ^5 + stats enable + server kb-importer kb-importer:8080 check resolvers docker_resolver resolve-prefer ipv4 + backend rest-backend-nodes mode http log global diff --git a/docker/kb-importer/Dockerfile b/docker/kb-importer/Dockerfile index b3419954f..9fb78174b 100644 --- a/docker/kb-importer/Dockerfile +++ b/docker/kb-importer/Dockerfile @@ -1,4 +1,5 @@ -FROM openjdk:11-jre-slim +# https://hub.docker.com/layers/eclipse-temurin/library/eclipse-temurin/11.0.15_10-jre/images/sha256-1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3?context=explore +FROM eclipse-temurin@sha256:1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3 LABEL maintainer="steady-dev@eclipse.org" @@ -6,7 +7,7 @@ ARG VULAS_RELEASE RUN apt-get update \ && apt-get install -y --no-install-recommends \ - openssl wget tar git cron bash gettext\ + openssl wget tar git cron bash gettext curl \ && rm -rf /var/lib/apt/lists/* \ && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false @@ -15,10 +16,12 @@ WORKDIR /kb-importer RUN wget https://github.com/SAP/project-kb/releases/download/v0.6.18/kaybee-0.6.18_linux-amd64 -O kaybee RUN chmod +x kaybee -COPY kb-importer-$VULAS_RELEASE-jar-with-dependencies.jar kb-importer.jar +COPY kb-importer-$VULAS_RELEASE.jar kb-importer.jar RUN chmod +x kb-importer.jar -COPY kb-importer.sh start.sh /kb-importer/ -RUN chmod +x /kb-importer/kb-importer.sh /kb-importer/start.sh +COPY run.sh /kb-importer/ +RUN chmod +x /kb-importer/run.sh -ENTRYPOINT ["sh","/kb-importer/start.sh"] +EXPOSE 8080 + +ENTRYPOINT ["sh","/kb-importer/run.sh"] diff --git a/docker/kb-importer/conf/kaybeeconf.yaml.sample b/docker/kb-importer/conf/kaybeeconf.yaml.sample index 7613cc13b..7310ae5b6 100644 --- a/docker/kb-importer/conf/kaybeeconf.yaml.sample +++ b/docker/kb-importer/conf/kaybeeconf.yaml.sample @@ -29,202 +29,6 @@ export: # each: | # {{ if .Fixes }} {{ .VulnerabilityID}},{{ (index (index .Fixes 0).Commits 0).RepositoryURL }}{{end}} # post: - - target: steady - filename: steady.sh - pre: | - #!/bin/bash - - # ------------------------------------------------ - # Import script for Eclipse Steady - # ------------------------------------------------ - - ##-------- E D I T T H I S S E C T I O N -------- - ## - ## COMMENT OUT THE NEXT LINE AND EDIT THE FOLLOWING LINES - #echo "Please configure the necessary variables in the script and try again" && exit 1 - - # Your user token (required to upload vulnerability data to Steady backend) - USER_TOKEN=$BACKEND_BUGS_TOKEN - - # Backend URL of your Steady instance - BACKEND_URL=$BACKEND_SERVICE_URL - - # URL of the CIA service of your Steady instance - CIA_URL=$CIA_SERVICE_URL - - # This must be the absolute path to the kb-importer jar file - KB_IMPORTER_PATH="/kb-importer/data/kb-importer.jar" - - # Skip repos clone and thus vulnerabilities requiring it - SKIP_CLONE=$KB_IMPORTER_SKIP_CLONE - ##---------------------- E N D -------------------- - - ANALYZER_CMD="java -Dvulas.shared.backend.header.X-Vulas-Client-Token=$USER_TOKEN -Dvulas.shared.cia.serviceUrl=$CIA_URL -Dvulas.shared.backend.serviceUrl=$BACKEND_URL -jar $KB_IMPORTER_PATH -u" - LOCAL_CLONES_DIR=$KB_IMPORTER_CLONE_FOLDER - - folder_for_repo(){ - X=$1 - - # remove trailing slash - X=${X%/} - - # remove everything until '://' is matched - X=${X#*:\/\/} - - # replace _ for . - X=${X//./_} - - # replace _ for / - X=${X//\//_} - echo ${LOCAL_CLONES_DIR}/$X - } - - clone_once(){ - DIR=$(folder_for_repo $1) - - if [ -d $DIR ]; - then - echo "Folder $DIR exists, skipping git clone" - else - echo "Cloning $1 to folder $DIR" - git clone $1 $DIR - fi - } - - #make_vuln_metadata(){ - # vulnerability_id="$1" - # description="$2" - # links="$3" - # - # [ -d $vulnerability_id ] || mkdir ./$vulnerability_id - # > $vulnerability_id/meta.properties - # echo "vulnerability_id=$vulnerability_id" >> $vulnerability_id/meta.properties - # echo "description=$description" >> $vulnerability_id/meta.properties - # echo "links=$links" >> $vulnerability_id/meta.properties - #} - - create_meta_from_tar(){ - repo_url=$1 - commit_id=$2 - branch=$3 - vulnerability_id=$4 - source_path=$5 - - if [ -f $vulnerability_id/$commit_id/metadata.json ] - then - return - fi - - [ -d $vulnerability_id/$commit_id ] || mkdir -p $vulnerability_id/$commit_id - if [ -f $source_path/changed-source-code.tar.gz ] - then - timestamp=`cat $vulnerability_id/$commit_id/timestamp` - echo "{" > $vulnerability_id/$commit_id/metadata.json - echo " \"repository\" : \"$repo_url\"," >> $vulnerability_id/$commit_id/metadata.json - echo " \"branch\" : \"$branch\"," >> $vulnerability_id/$commit_id/metadata.json - echo " \"timestamp\" : \"$timestamp\"," >> $vulnerability_id/$commit_id/metadata.json - echo " \"commit_id\" : \"$commit_id\"" >> $vulnerability_id/$commit_id/metadata.json - echo "}" >> $vulnerability_id/$commit_id/metadata.json - rm $vulnerability_id/$commit_id/timestamp - fi - return - } - - clone_and_create_meta(){ - repo_url=$1 - commit_id=$2 - branch=$3 - vulnerability_id=$4 - source_path=$5 - - if [ -f $vulnerability_id/$commit_id/metadata.json ] - then - return - fi - - clone_once $repo_url - repo_dir=$(folder_for_repo $repo_url) - - timestamp=$(git -C $repo_dir show --no-patch --no-notes --pretty='%at' $commit_id) - - echo "{" > $vulnerability_id/$commit_id/metadata.json - echo " \"repository\" : \"$repo_url\"," >> $vulnerability_id/$commit_id/metadata.json - echo " \"branch\" : \"$branch\"," >> $vulnerability_id/$commit_id/metadata.json - echo " \"timestamp\" : \"$timestamp\"," >> $vulnerability_id/$commit_id/metadata.json - echo " \"commit_id\" : \"$commit_id\"" >> $vulnerability_id/$commit_id/metadata.json - echo "}" >> $vulnerability_id/$commit_id/metadata.json - - echo "dir=$repo_dir" - echo "pwd=`pwd`" - - # cd repository - for F in $(git -C $repo_dir diff --name-only $commit_id^..$commit_id); - do - - echo "repo_dir=$repo_dir" - echo "pwd=`pwd`" - - echo "Extracting file: $F" - [ -d $vulnerability_id/$commit_id/before/$(dirname $F) ] || mkdir -p $vulnerability_id/$commit_id/before/$(dirname $F) - [ -d $vulnerability_id/$commit_id/after/$(dirname $F) ] || mkdir -p $vulnerability_id/$commit_id/after/$(dirname $F) - - if ( git -C $repo_dir cat-file -e $commit_id~1:$F &> /dev/null ) - then - git -C $repo_dir show $commit_id~1:$F > $vulnerability_id/$commit_id/before/$F - fi - - if ( git -C $repo_dir cat-file -e $commit_id:$F &> /dev/null ) - then - git -C $repo_dir show $commit_id:$F > $vulnerability_id/$commit_id/after/$F - fi - done - } - - each: |+ - # ----------------------------------------------- - # Analyzing vulnerability {{ .VulnerabilityID}} - # ----------------------------------------------- - {{ if .VulnerabilityID }} - [ -d ./{{ .VulnerabilityID }} ] || mkdir ./{{ .VulnerabilityID }} - {{ $source_path := .Metadata.LocalPath }} - [ -f {{ $source_path }}/changed-source-code.tar.gz ] && tar -xf {{ $source_path }}/changed-source-code.tar.gz -C ./{{ .VulnerabilityID }} - - cat << 'EOM' > ./{{ .VulnerabilityID }}/metadata.json - {{ .ToJSON }} - EOM - - if [ -f {{ $source_path }}/changed-source-code.tar.gz ] ; - then - # Create the metadata from the tarball cloned previously - {{ if .Fixes}}{{ $description := or ((index .Notes 0).Text) "" }} - {{ $vuln := .VulnerabilityID}}{{ $repo := (index (index .Fixes 0).Commits 0).RepositoryURL }} - {{ range $f := .Fixes }}{{ range .Commits }}create_meta_from_tar {{$repo}} {{.ID}} {{$f.ID}} {{ $vuln }} {{ $source_path }} - {{end}}{{end}} - {{end}} - $ANALYZER_CMD -d ./{{ .VulnerabilityID }} - : - else - if [ ! "${KB_IMPORTER_SKIP_CLONE}" == "True" ]; - then - # Create the metadata after cloning the repo of the affected package and checking out every commit - {{ if .Fixes}}{{ $description := or ((index .Notes 0).Text) "" }} - #make_vuln_metadata {{ .VulnerabilityID }} '{{ JoinNotes . }}' {{ LinksAsCSV . }} - {{ $vuln := .VulnerabilityID}}{{ $repo := (index (index .Fixes 0).Commits 0).RepositoryURL }} - {{ range $f := .Fixes }}{{ range .Commits }}clone_and_create_meta {{$repo}} {{.ID}} {{$f.ID}} {{ $vuln }} {{ $source_path }} - {{end}}{{end}} - {{else}} - # This vulnerability has no fix-commits - : - {{end}} - $ANALYZER_CMD -d ./{{ .VulnerabilityID }} - fi - fi - {{end}} - - post: |- - # ------------------------------------------------ - # This script was generated with KayBee - # ------------------------------------------------ - target: xml filename: vulnerabilities.xml pre: | @@ -251,4 +55,4 @@ export: post: | - + \ No newline at end of file diff --git a/docker/kb-importer/kb-importer.sh b/docker/kb-importer/kb-importer.sh deleted file mode 100644 index f9fe7e1b9..000000000 --- a/docker/kb-importer/kb-importer.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -construct_kaybee_pull_folder(){ - X=$1 - - # remove trailing slash - X=${X%/} - - # remove everything until '://' is matched - X=${X#*:\/\/} - - # Remove the longest matching suffix pattern - HOST=${X%%/*} - - # Remove shortest matching prefix pattern. - PATH=${X#*\/} - - # replace . for / - PATH=${PATH//\//.} - - echo ${HOST}_${PATH} -} - -#if [ ! -f /kb-importer/data/running ] -#then -# touch /kb-importer/data/running - - #kaybee update - cd /kb-importer/data - ./kaybee update --force - - #run kaybee import for kaybeeconf.yaml (as it contains the substituted env variables for the source repo and branch) - echo `date` " Running Kaybee Import" - ./kaybee pull -c ../conf/kaybeeconf.yaml -# As we cannot configure the destination folder of kaybee pull (for now), we explicitly copy the resulting folder to the configured one and skip kaybee merge as we only have 1 source configurable - cp -r .kaybee/repositories/$(construct_kaybee_pull_folder $KB_IMPORTER_STATEMENTS_REPO)_$KB_IMPORTER_STATEMENTS_BRANCH/statements/. $KB_IMPORTER_STATEMENTS_FOLDER/ -# echo `date` " Running Kaybee Merge" >> job.log 2>&1 -# ./kaybee merge -s -c ../conf/kaybeeconf.yaml -# echo `date` " Kaybee Merge Done" >> job.log 2>&1 - ./kaybee export -t steady -c ../conf/kaybeeconf.yaml -f $KB_IMPORTER_STATEMENTS_FOLDER - chmod +x steady.sh - ./steady.sh - echo `date` " Kaybee Import Done" -# rm /kb-importer/data/running -#else -# echo `date` " Kaybee Import already Running" -#fi diff --git a/docker/kb-importer/run.sh b/docker/kb-importer/run.sh new file mode 100644 index 000000000..397d5849d --- /dev/null +++ b/docker/kb-importer/run.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Home directory of kb-importer +home="/kb-importer" + +mkdir -p $home/data + +echo "Statements repo: " $KB_IMPORTER_STATEMENTS_REPO +echo "Statements branch: " $KB_IMPORTER_STATEMENTS_BRANCH +echo "Statements folder: " $KB_IMPORTER_STATEMENTS_FOLDER +echo "Skip clones: " $KB_IMPORTER_SKIP_CLONE + +# Substitute env variables used by kaybee in kaybeeconf.yaml +sed "s|KB_IMPORTER_STATEMENTS_REPO|$KB_IMPORTER_STATEMENTS_REPO|g" $home/conf/kaybeeconf.yaml.sample > $home/conf/kaybeeconf.yaml +sed -i "s|KB_IMPORTER_STATEMENTS_BRANCH|$KB_IMPORTER_STATEMENTS_BRANCH|g" $home/conf/kaybeeconf.yaml + +# Adding certs +certs=`ls $home/certs | grep -v readme.txt` +for cert in $certs; do + keytool -import -alias $cert -storepass changeit -keystore /usr/lib/jvm/java-1.8-openjdk/jre/lib/security/cacerts -file $home/certs/$cert -noprompt +done + +java -Dvulas.shared.backend.header.X-Vulas-Client-Token=$BACKEND_BUGS_TOKEN \ + -Dvulas.shared.cia.serviceUrl=$CIA_SERVICE_URL \ + -Dvulas.shared.backend.serviceUrl=$BACKEND_SERVICE_URL \ + -jar $home/kb-importer.jar | tee $home/data/analyzer_logs.txt diff --git a/docker/kb-importer/start.sh b/docker/kb-importer/start.sh deleted file mode 100644 index 775bc4b21..000000000 --- a/docker/kb-importer/start.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -mkdir -p /kb-importer/data -cd /kb-importer/data -if [ -d $KB_IMPORTER_CLONE_FOLDER ] && [ ! -z $KB_IMPORTER_CLONE_FOLDER ]; then - mkdir -p $KB_IMPORTER_CLONE_FOLDER -fi -if [ -f /kb-importer/kb-importer.jar ]; then - mv /kb-importer/kb-importer.jar /kb-importer/kaybee /kb-importer/data -fi - -#substitute env variables used by kaybee in kaybeeconf.yaml -sed "s|KB_IMPORTER_STATEMENTS_REPO|$KB_IMPORTER_STATEMENTS_REPO|g" ../conf/kaybeeconf.yaml.sample > ../conf/kaybeeconf.yaml -sed -i "s|KB_IMPORTER_STATEMENTS_BRANCH|$KB_IMPORTER_STATEMENTS_BRANCH|g" ../conf/kaybeeconf.yaml - -echo "Statements repo: " $KB_IMPORTER_STATEMENTS_REPO -echo "Statements branch: " $KB_IMPORTER_STATEMENTS_BRANCH -echo "Statements folder: " $KB_IMPORTER_STATEMENTS_FOLDER -echo "Clones folder: " $KB_IMPORTER_CLONE_FOLDER -echo "Skip clones: " $KB_IMPORTER_SKIP_CLONE - -./kaybee update --force - -#Adding certs -certs=`ls /kb-importer/certs | grep -v readme.txt` -for cert in $certs; do - keytool -import -alias $cert -storepass changeit -keystore /usr/lib/jvm/java-1.8-openjdk/jre/lib/security/cacerts -file /kb-importer/certs/$cert -noprompt -done - -#Wait for backend to start -sleep 40 - -#Run initial import -./../kb-importer.sh - -#create a cron job kaybeeconf.yaml -crontab -l > tmpcron -if ! cat tmpcron | grep "kb-importer.sh" -then - if [ -z "$KB_IMPORTER_CRON_HOUR" ] - then - echo "0 0 * * * PATH=$PATH BACKEND_SERVICE_URL=$BACKEND_SERVICE_URL KB_IMPORTER_STATEMENTS_FOLDER=$KB_IMPORTER_STATEMENTS_FOLDER KB_IMPORTER_STATEMENTS_BRANCH=$KB_IMPORTER_STATEMENTS_BRANCH KB_IMPORTER_STATEMENTS_REPO=$KB_IMPORTER_STATEMENTS_REPO KB_IMPORTER_CLONE_FOLDER=$KB_IMPORTER_CLONE_FOLDER KB_IMPORTER_SKIP_CLONE=$KB_IMPORTER_SKIP_CLONE /kb-importer/kb-importer.sh >> /kb-importer/cron.log 2>&1" >> tmpcron - else - echo "0 " "$KB_IMPORTER_CRON_HOUR" " * * * PATH=$PATH BACKEND_SERVICE_URL=$BACKEND_SERVICE_URL KB_IMPORTER_STATEMENTS_FOLDER=$KB_IMPORTER_STATEMENTS_FOLDER KB_IMPORTER_STATEMENTS_BRANCH=$KB_IMPORTER_STATEMENTS_BRANCH KB_IMPORTER_STATEMENTS_REPO=$KB_IMPORTER_STATEMENTS_REPO KB_IMPORTER_CLONE_FOLDER=$KB_IMPORTER_CLONE_FOLDER KB_IMPORTER_SKIP_CLONE=$KB_IMPORTER_SKIP_CLONE /kb-importer/kb-importer.sh >> /kb-importer/cron.log 2>&1" >> tmpcron - fi -fi -crontab tmpcron -echo "cron job created." -rm tmpcron -cron -f diff --git a/docker/patch-lib-analyzer/Dockerfile b/docker/patch-lib-analyzer/Dockerfile index b5198975d..911dd04f0 100644 --- a/docker/patch-lib-analyzer/Dockerfile +++ b/docker/patch-lib-analyzer/Dockerfile @@ -1,4 +1,5 @@ -FROM openjdk:11-jre-slim +# https://hub.docker.com/layers/eclipse-temurin/library/eclipse-temurin/11.0.15_10-jre/images/sha256-1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3?context=explore +FROM eclipse-temurin@sha256:1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3 LABEL maintainer="steady-dev@eclipse.org" @@ -10,9 +11,9 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false -COPY patch-lib-analyzer-${VULAS_RELEASE}-jar-with-dependencies.jar /vulas/patch-lib-analyzer.jar -COPY run.sh /vulas/run.sh +COPY patch-lib-analyzer-${VULAS_RELEASE}-jar-with-dependencies.jar /steady/patch-lib-analyzer.jar +COPY run.sh /steady/run.sh -RUN chmod +x /vulas/run.sh +RUN chmod +x /steady/run.sh -CMD ["/vulas/run.sh"] +CMD ["/steady/run.sh"] diff --git a/docker/patch-lib-analyzer/run.sh b/docker/patch-lib-analyzer/run.sh index d40cb4fca..015c61e77 100755 --- a/docker/patch-lib-analyzer/run.sh +++ b/docker/patch-lib-analyzer/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#Wait for backend to start and kb-importer to insert data +# Wait for backend to start and kb-importer to insert data sleep 300 java \ @@ -10,4 +10,4 @@ java \ -Dhttps.proxyHost=$HTTPS_PROXY_HOST \ -Dhttps.proxyPort=$HTTPS_PROXY_PORT \ -Dspring.profiles.active=docker \ - -jar /vulas/patch-lib-analyzer.jar $PATCHEVAL_OPTS + -jar /steady/patch-lib-analyzer.jar $PATCHEVAL_OPTS diff --git a/docker/postgresql/docker-entrypoint-initdb.d/10-vulas-setup.sh b/docker/postgresql/conf/10-vulas-setup.sh similarity index 100% rename from docker/postgresql/docker-entrypoint-initdb.d/10-vulas-setup.sh rename to docker/postgresql/conf/10-vulas-setup.sh diff --git a/docker/rest-backend/Dockerfile b/docker/rest-backend/Dockerfile index f1d2e1e9c..0405e4afd 100644 --- a/docker/rest-backend/Dockerfile +++ b/docker/rest-backend/Dockerfile @@ -1,4 +1,5 @@ -FROM openjdk:11-jre-slim +# https://hub.docker.com/layers/eclipse-temurin/library/eclipse-temurin/11.0.15_10-jre/images/sha256-1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3?context=explore +FROM eclipse-temurin@sha256:1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3 LABEL maintainer="steady-dev@eclipse.org" @@ -10,12 +11,12 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false -COPY rest-backend-$VULAS_RELEASE.jar /vulas/rest-backend.jar -COPY run.sh /vulas/run.sh +COPY rest-backend-$VULAS_RELEASE.jar /steady/rest-backend.jar +COPY run.sh /steady/run.sh RUN touch /$VULAS_RELEASE EXPOSE 8091 -RUN chmod +x /vulas/run.sh +RUN chmod +x /steady/run.sh -CMD ["/vulas/run.sh"] +CMD ["/steady/run.sh"] diff --git a/docker/rest-backend/run.sh b/docker/rest-backend/run.sh index 0ba953a87..64a02b661 100755 --- a/docker/rest-backend/run.sh +++ b/docker/rest-backend/run.sh @@ -20,4 +20,4 @@ java \ -Dvulas.jira.pwd=$JIRA_PASSWORD \ $FLYWAY_OPTS \ -Dspring.profiles.active=docker \ - -jar /vulas/rest-backend.jar + -jar /steady/rest-backend.jar diff --git a/docker/rest-lib-utils/Dockerfile b/docker/rest-lib-utils/Dockerfile index e1401ffeb..ffaf9909e 100644 --- a/docker/rest-lib-utils/Dockerfile +++ b/docker/rest-lib-utils/Dockerfile @@ -1,12 +1,13 @@ -FROM openjdk:11-jre-slim +# https://hub.docker.com/layers/eclipse-temurin/library/eclipse-temurin/11.0.15_10-jre/images/sha256-1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3?context=explore +FROM eclipse-temurin@sha256:1543416e05e9fde8ffede76cd5f0955b640d7159bdbff8574eed6560a98e4ad3 LABEL maintainer="steady-dev@eclipse.org" ARG VULAS_RELEASE -COPY rest-lib-utils-${VULAS_RELEASE}.jar /vulas/rest-lib-utils.jar +COPY rest-lib-utils-${VULAS_RELEASE}.jar /steady/rest-lib-utils.jar RUN touch /$VULAS_RELEASE EXPOSE 8092 -CMD java -Dhttp.nonProxyHosts=${NON_PROXY_HOSTS} -Dhttps.nonProxyHosts=${NON_PROXY_HOSTS} -Dhttps.proxyHost=${HTTPS_PROXY_HOST} -Dhttps.proxyPort=${HTTP_PROXY_PORT} -Dhttp.proxyHost=${HTTP_PROXY_HOST} -Dhttp.proxyPort=${HTTP_PROXY_PORT} -jar /vulas/rest-lib-utils.jar +CMD java -Dhttp.nonProxyHosts=${NON_PROXY_HOSTS} -Dhttps.nonProxyHosts=${NON_PROXY_HOSTS} -Dhttps.proxyHost=${HTTPS_PROXY_HOST} -Dhttps.proxyPort=${HTTP_PROXY_PORT} -Dhttp.proxyHost=${HTTP_PROXY_HOST} -Dhttp.proxyPort=${HTTP_PROXY_PORT} -jar /steady/rest-lib-utils.jar diff --git a/docker/setup-steady.sh b/docker/setup-steady.sh index c661f2e22..1fb7bf5f6 100755 --- a/docker/setup-steady.sh +++ b/docker/setup-steady.sh @@ -48,23 +48,23 @@ setup (){ # Create directories #mkdir -p $DIR/certs for s in $conf_services; do - mkdir -p $DIR/conf/$s + mkdir -p $DIR/$s/conf done for s in $data_services; do - mkdir -p $DIR/data/$s + mkdir -p $DIR/$s/data done # Download all necessary files - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/docker-compose-new.yml -o ./$DIR/docker-compose.yml - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/.env.sample -o ./$DIR/.env - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/start-steady.sh -o ./$DIR/start-steady.sh + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/docker-compose.yml -o ./$DIR/docker-compose.yml + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/.env.sample -o ./$DIR/.env + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/start-steady.sh -o ./$DIR/start-steady.sh chmod 744 ./$DIR/start-steady.sh - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/haproxy/conf/haproxy.cfg -o ./$DIR/conf/haproxy/haproxy.cfg - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/postgresql/docker-entrypoint-initdb.d/10-vulas-setup.sh -o ./$DIR/conf/postgresql/10-vulas-setup.sh - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/cache/nginx.conf -o ./$DIR/conf/cache/nginx.conf - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/kb-importer/conf/kaybeeconf.yaml.sample -o ./$DIR/conf/kb-importer/kaybeeconf.yaml.sample - curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/rest-backend/conf/restbackend.properties -o ./$DIR/conf/rest-backend/restbackend.properties + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/haproxy/conf/haproxy.cfg -o ./$DIR/haproxy/conf/haproxy.cfg + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/postgresql/conf/10-vulas-setup.sh -o ./$DIR/postgresql/conf/10-vulas-setup.sh + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/cache/conf/nginx.conf -o ./$DIR/cache/conf/nginx.conf + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/kb-importer/conf/kaybeeconf.yaml.sample -o ./$DIR/kb-importer/conf/kaybeeconf.yaml.sample + curl -s https://raw.githubusercontent.com/eclipse/steady/$TAG/docker/rest-backend/conf/restbackend.properties -o ./$DIR/rest-backend/conf/restbackend.properties # Create default configuration in user's home directory home_config="$HOME/.steady.properties" diff --git a/docker/start-steady.sh b/docker/start-steady.sh index 0c23bdbc6..80f7769df 100755 --- a/docker/start-steady.sh +++ b/docker/start-steady.sh @@ -96,10 +96,18 @@ if [[ $rc == 0 ]]; then exit 1 fi +# If run from the cloned repo, the file docker-compose.build.yaml can be used to +# create new Docker images for snapshot versions. When run after +# steady-setup.sh, the images will be downloaded from Docker Hub. +build="" +if [[ -f docker-compose.build.yml ]]; then + build="-f docker-compose.build.yml" +fi + # Start different sets of services case $SERVICES in none) - docker-compose -f ./docker-compose.yml stop + docker-compose -f ./docker-compose.yml $build stop rc=$? if [[ $rc == 0 ]]; then printf "Stopped all of Steady's Docker Compose services\n" @@ -111,7 +119,7 @@ case $SERVICES in core) stop_ui stop_vdb - docker-compose -f ./docker-compose.yml up -d --build + docker-compose -f ./docker-compose.yml $build up -d --build rc=$? if [[ $rc == 0 ]]; then printf "Started Steady's core Docker Compose services\n" @@ -122,7 +130,7 @@ case $SERVICES in ;; ui) stop_vdb - docker-compose -f ./docker-compose.yml --profile ui up -d --build + docker-compose -f ./docker-compose.yml $build --profile ui up -d --build rc=$? if [[ $rc == 0 ]]; then printf "Started Steady's core and UI Docker Compose services\n" @@ -133,7 +141,7 @@ case $SERVICES in ;; vdb) stop_ui - docker-compose -f ./docker-compose.yml --profile vdb up -d --build + docker-compose -f ./docker-compose.yml $build --profile vdb up -d --build rc=$? if [[ $rc == 0 ]]; then printf "Started Steady's core and vdb Docker Compose services\n" @@ -143,7 +151,7 @@ case $SERVICES in fi ;; all) - docker-compose -f ./docker-compose.yml --profile ui --profile vdb up -d --build + docker-compose -f ./docker-compose.yml $build --profile ui --profile vdb up -d --build rc=$? if [[ $rc == 0 ]]; then printf "Started all of Steady's Docker Compose services\n" diff --git a/docs/public/content/admin/tutorials/build.md b/docs/public/content/admin/tutorials/build.md index dc864fdae..2c6a433fd 100644 --- a/docs/public/content/admin/tutorials/build.md +++ b/docs/public/content/admin/tutorials/build.md @@ -43,6 +43,8 @@ docker run -it --rm -v ${PWD}/docker:/exporter --env-file ./docker/.env -e mvn_f > In case you are running behind a proxy you need to configure it in the `--build-arg` arguments. Check the [predefined `ARG`s](https://docs.docker.com/engine/reference/builder/#predefined-args) documentation to know more. +> In case you'd like to avoid downloading all dependencies by reusing your local Maven repository, add a volume as follows `-v "$HOME/.m2":/root/.m2`, see [here](https://hub.docker.com/_/maven/) for more information. + As a result, the folders `docker/` will contain compiled JARs (or WARs, depending on the component). The folder `docker/client-tools` will be populated with the JARs for client side tools (CLI, plugins, patchanalyzer). Additionally, you may want to make the artifacts available to the developers of your organization (e.g., through an internal Nexus or other artifact distribution system). @@ -58,13 +60,18 @@ You are now ready to run the system with the generated archives and create the D You can create and run containers from the generated images. ```sh -(cd docker && docker-compose -f docker-compose.yml -f docker-compose.build.yml up -d) +(cd docker && docker-compose -f docker-compose.yml -f docker-compose.build.yml --profile vdb --profile ui up -d) ``` To check everything started successfully, browse the page `http://localhost:8033/haproxy?stats`. All endpoints should appear as green. > `username` and `password` can be found in your `.env` file, be also advised that `rest-backend` could take more than 30 seconds to be ready to answer HTTP requests +Run the following to stop all containers. + +```sh +(cd docker && docker-compose -f docker-compose.yml -f docker-compose.build.yml --profile vdb --profile ui down) +``` --- Get going: diff --git a/kb-importer/pom.xml b/kb-importer/pom.xml old mode 100755 new mode 100644 index e4eacc46d..0b5243fc7 --- a/kb-importer/pom.xml +++ b/kb-importer/pom.xml @@ -1,48 +1,92 @@ - + 4.0.0 + - root - org.eclipse.steady - 3.2.5-SNAPSHOT + org.springframework.boot + spring-boot-starter-parent + 2.6.3 + + org.eclipse.steady kb-importer + 3.2.5-SNAPSHOT Knowledge Base Importer - + + Imports vulnerabilities from Project KB into Steady's database. + + + + 8 + true + true + true + + UTF-8 + UTF-8 + + + true + ${skip.install.deploy} + ${skip.install.deploy} + + + 2021-06-22T10:45:00Z + + + org.springframework.boot + spring-boot-starter-web + + - commons-cli - commons-cli + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-test + test + + + + + commons-io commons-io + 2.11.0 + + + + org.yaml + snakeyaml + 1.30 + org.eclipse.steady @@ -50,6 +94,7 @@ ${project.version} compile + org.eclipse.steady @@ -58,6 +103,12 @@ ${project.version} test + + junit + junit + 4.13.2 + + @@ -72,92 +123,140 @@ ${project.version} runtime - - com.github.package-url - packageurl-java - + com.jayway.jsonpath json-path 2.7.0 + + + + + prepare-release + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.3.0 + + + attach-javadocs + + jar + + + + + + org.cyclonedx + cyclonedx-maven-plugin + 2.6.2 + + + package + makeAggregateBom + + + + + + + + + + release + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + + + + + - org.apache.maven.plugins - maven-resources-plugin - - - - org.apache.maven.plugins - maven-shade-plugin - - - make-shade - package + org.codehaus.mojo + flatten-maven-plugin + 1.2.2 + + + flatten + process-resources - shade + flatten - - - true - true - - - false - true - jar-with-dependencies - - - - - org.eclipse.steady.kb.Main - ${project.version} - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - + clean - - - - - - org.apache.maven.plugins - maven-failsafe-plugin - - alphabetical - - - - integration-test - - integration-test - - - verify + flatten.clean + clean - verify + clean + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + org.apache.maven.surefire + surefire-junit4 + 3.0.0-M5 + + + diff --git a/kb-importer/src/main/java/org/eclipse/steady/kb/ImportCommand.java b/kb-importer/src/main/java/org/eclipse/steady/kb/ImportCommand.java new file mode 100644 index 000000000..6cf23f23f --- /dev/null +++ b/kb-importer/src/main/java/org/eclipse/steady/kb/ImportCommand.java @@ -0,0 +1,233 @@ +/** + * This file is part of Eclipse Steady. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors + */ +package org.eclipse.steady.kb; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; + +import org.apache.logging.log4j.Logger; +import org.eclipse.steady.backend.BackendConnectionException; +import org.eclipse.steady.backend.BackendConnector; +import org.eclipse.steady.kb.model.Vulnerability; +import org.eclipse.steady.kb.task.ExtractOrClone; +import org.eclipse.steady.kb.task.ImportAffectedLibraries; +import org.eclipse.steady.kb.task.ImportVulnerability; +import org.eclipse.steady.kb.util.Metadata; +import org.eclipse.steady.shared.util.FileUtil; +import org.eclipse.steady.shared.util.StopWatch; + +import com.github.packageurl.MalformedPackageURLException; + +/** + * Imports information pertaining to a single vulnerability statement. To do so, + * it calls 3 tasks in method {@link ImportCommand#run()}: + * {@link ExtractOrClone}, {@link ImportVulnerability} and + * {@link ImportAffectedLibraries}. + */ +public class ImportCommand implements Runnable { + + /** The file name of statements coming from Project KB. */ + public static final String STATEMENT_YAML = "statement.yaml"; + + /** The file name of tarballs coming from Project KB, and which contain the + * source code changes created by the respective fix commits. */ + public static final String SOURCE_TAR = "changed-source-code.tar.gz"; + + /** Constant VERBOSE_OPTION="v" */ + public static final String VERBOSE_OPTION = "v"; + + /** Constant UPLOAD_CONSTRUCT_OPTION="u" */ + public static final String UPLOAD_CONSTRUCT_OPTION = "u"; + + /** Constant SKIP_CLONE_OPTION="u" */ + public static final String SKIP_CLONE_OPTION = "u"; + + /** Constant OVERWRITE_OPTION="o" */ + public static final String OVERWRITE_OPTION = "o"; + + /** Constant DELETE_OPTION="del" */ + public static final String DELETE_OPTION = "del"; + + /** Constant DIRECTORY_OPTION="d" */ + public static final String DIRECTORY_OPTION = "d"; + + /** Constant TIME_REFETCH_ALL_OPTION="t" */ + public static final String TIME_REFETCH_ALL_OPTION = "t"; + + /** Constant DELETE="del" */ + public static final String DELETE = "del"; + + /** Constant SEQUENTIAL="seq" */ + public static final String SEQUENTIAL = "seq"; + + private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(); + + private StopWatch stopWatch = null; + private Path vulnDir; + private String vulnId; + private HashMap args; + + /** + * The {@link Manager} that started the command. Used to reflect the status of + * the import and maintain a list of failed imports. + */ + Manager manager; + + /** + *

Constructor for ImportCommand.

+ * + * @param manager a {@link org.eclipse.steady.kb.Manager} object + * @param args a {@link java.util.HashMap} object + */ + public ImportCommand( + Manager manager, HashMap args) { + this.manager = manager; + this.vulnDir = Paths.get((String) args.get(DIRECTORY_OPTION)); + this.vulnId = vulnDir.getFileName().toString(); + this.args = args; + this.stopWatch = new StopWatch(this.vulnId); + } + + /** + *

Getter for the field vulnId.

+ * + * @return a {@link java.lang.String} object + */ + public String getVulnId() { + return this.vulnId; + } + + /** {@inheritDoc} */ + @Override + public void run() { + this.stopWatch.start(); + manager.setVulnStatus(vulnId, Manager.VulnStatus.STARTING); + + BackendConnector backend_connector = BackendConnector.getInstance(); + + // Does the vulnerability already exist? + boolean bugExists = false; + try { + bugExists = backend_connector.isBugExisting(vulnId); + } catch (BackendConnectionException e) { + manager.setVulnStatus(vulnId, Manager.VulnStatus.FAILED_CONNECTION); + manager.addFailure(vulnId, e); + this.stopWatch.stop(e); + return; + } + + // Override or not? + Boolean overwrite = false; + if (args.containsKey(OVERWRITE_OPTION)) { + overwrite = (Boolean) args.get(OVERWRITE_OPTION); + } + if (bugExists) { + if (overwrite) { + args.put(DELETE, true); + log.info("Bug [{}] already exists in backend and will be overwritten", vulnId); + } else { + log.info("Bug [{}] already exists in backend, analysis will be skipped", vulnId); + manager.setVulnStatus(vulnId, Manager.VulnStatus.IMPORTED); + this.stopWatch.stop(); + return; + } + } + else { + manager.addNewVulnerability(vulnId); + log.info("Bug [{}] does not exist in backend", vulnId); + } + + Path statementPath = this.vulnDir.resolve(STATEMENT_YAML); + + // statement.yaml does not exist? This should not happen, because the + // Manager only picks directories that contain a statement.yaml file. + if (!FileUtil.isAccessibleFile(statementPath)) { + ImportCommand.log.error("Cannot read [" + statementPath + "]"); + manager.setVulnStatus(vulnId, Manager.VulnStatus.MALFORMED_INPUT); + this.stopWatch.stop(); + return; + } + // Proceed with the import + else { + + // Read statement.yaml + Vulnerability vuln; + try { + vuln = Metadata.getFromYaml(statementPath.toString()); + } catch (IOException e) { + this.stopWatch.stop(e); + return; + } + + // Statement does not have commits nor affected libs? + if ((vuln.getCommits() == null || vuln.getCommits().size() == 0) + && (vuln.getArtifacts() == null || vuln.getArtifacts().size() == 0)) { + log.warn("No fix commits or affected artifacts for vulnerability [" + vuln.getVulnId() + "]"); + manager.setVulnStatus(vuln.getVulnId(), Manager.VulnStatus.MALFORMED_INPUT); + this.stopWatch.stop(); + return; + } + + // Extract source code tarball (if any) or clone repo + else { + ExtractOrClone extractOrClone = + new ExtractOrClone( + this.manager, + vuln, + new File(this.vulnDir.toString()), + (boolean) args.get(SKIP_CLONE_OPTION)); + extractOrClone.execute(); + this.stopWatch.lap("Cloned repo or extracted source code tarball"); + } + + if (manager.getVulnStatus(vuln.getVulnId()) != Manager.VulnStatus.FAILED_EXTRACT_OR_CLONE + && manager.getVulnStatus(vuln.getVulnId()) != Manager.VulnStatus.SKIP_CLONE) { + + manager.setVulnStatus(vuln.getVulnId(), Manager.VulnStatus.IMPORTING); + + try { + ImportVulnerability importVulnerability = new ImportVulnerability(); + importVulnerability.execute(vuln, args, backend_connector); + this.stopWatch.lap("Imported change list using the fix-commits"); + } catch (IOException | BackendConnectionException e) { + manager.setVulnStatus(vuln.getVulnId(), Manager.VulnStatus.FAILED_IMPORT_VULN); + manager.addFailure(vuln.getVulnId(), e); + this.stopWatch.stop(e); + return; + } + + try { + ImportAffectedLibraries importAffectedLibraries = new ImportAffectedLibraries(); + importAffectedLibraries.execute(vuln, args, backend_connector); + this.stopWatch.lap("Imported affected libraries"); + } catch (IOException | MalformedPackageURLException | BackendConnectionException e) { + manager.setVulnStatus(vuln.getVulnId(), Manager.VulnStatus.FAILED_IMPORT_LIB); + manager.addFailure(vuln.getVulnId(), e); + this.stopWatch.stop(e); + return; + } + manager.setVulnStatus(vuln.getVulnId(), Manager.VulnStatus.IMPORTED); + } + this.stopWatch.stop(); + } + } +} diff --git a/kb-importer/src/main/java/org/eclipse/steady/kb/Manager.java b/kb-importer/src/main/java/org/eclipse/steady/kb/Manager.java new file mode 100644 index 000000000..a63b17721 --- /dev/null +++ b/kb-importer/src/main/java/org/eclipse/steady/kb/Manager.java @@ -0,0 +1,443 @@ +/** + * This file is part of Eclipse Steady. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors + */ +package org.eclipse.steady.kb; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.http.conn.HttpHostConnectException; +import org.eclipse.steady.backend.BackendConnectionException; +import org.eclipse.steady.core.util.CoreConfiguration; +import org.eclipse.steady.shared.util.DirWithFileSearch; +import org.eclipse.steady.shared.util.FileUtil; +import org.eclipse.steady.shared.util.ProcessWrapper; +import org.eclipse.steady.shared.util.ProcessWrapperException; +import org.eclipse.steady.shared.util.StopWatch; +import org.eclipse.steady.shared.util.ThreadUtil; +import org.eclipse.steady.shared.util.VulasConfiguration; + +import com.google.gson.Gson; +/** + * Creates and executes threads for processing each vulnerability. + * Keeps track of the state of each one of them. + */ +public class Manager { + + private static final org.apache.logging.log4j.Logger log = + org.apache.logging.log4j.LogManager.getLogger(); + + final String kaybeeBinaryPath = + VulasConfiguration.getGlobal() + .getConfiguration() + .getString("vulas.kb-importer.kaybeeBinaryPath"); + + final String kaybeeConfPath = + VulasConfiguration.getGlobal() + .getConfiguration() + .getString("vulas.kb-importer.kaybeeConfPath"); + + /** + * The folder into which kaybee pulls the statements. + * Should be changed as soon as kaybee merge is properly implemented. + */ + private static final String KAYBEE_STMTS_PATH = ".kaybee/repositories/github.com_sap.project-kb_vulnerability-data/statements"; + + /** + * The data folder inside kb-importer's Docker container. + */ + private static final String IMPORT_STMTS_PATH = + VulasConfiguration.getGlobal() + .getConfiguration() + .getString("vulas.kb-importer.statementsPath"); + + private ThreadPoolExecutor executor; + + private static Map vulnerabilitiesStatus = new HashMap(); + + private static Set newVulnerabilities = new LinkedHashSet(); + + // pairs of vulnId and reason for failure + private static Map failures = new HashMap(); + + Map repoLocks = new HashMap(); + + private StopWatch stopWatch = null; + + private Path tmpDir = null; + + public enum VulnStatus { + NOT_STARTED, + STARTING, + EXTRACTING, + CLONING, + IMPORTING, + IMPORTED, + FAILED_EXTRACT_OR_CLONE, + FAILED_CONNECTION, + SKIP_CLONE, + FAILED_IMPORT_LIB, + FAILED_IMPORT_VULN, + MALFORMED_INPUT + } + + /** + *

Constructor for Manager.

+ * + * @param backendConnector a {@link org.eclipse.steady.backend.BackendConnector} object + */ + public Manager() { + this.createNewExecutor(); + try { + this.tmpDir = FileUtil.createTmpDir("import"); + } catch (IOException e) { + log.error("Error creating temp dir: " + e.getMessage()); + } + } + + /** + *

createNewExecutor.

+ */ + public void createNewExecutor() { + this.executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(ThreadUtil.getNoThreads()); + } + + /** + * Adds a vulnerability that does not yet exist in the backend. Called by {@link ImportCommand#run()}. + * + * @param vulnId a {@link java.lang.String} object + */ + public void addNewVulnerability(String vulnId) { + newVulnerabilities.add(vulnId); + } + + /** + *

setVulnStatus.

+ * + * @param vulnId a {@link java.lang.String} object + * @param vulnStatus a {@link org.eclipse.steady.kb.Manager.VulnStatus} object + */ + public void setVulnStatus(String vulnId, VulnStatus vulnStatus) { + vulnerabilitiesStatus.put(vulnId, vulnStatus); + } + + /** + *

getVulnStatus.

+ * + * @param vulnId a {@link java.lang.String} object + * @return a {@link org.eclipse.steady.kb.Manager.VulnStatus} object + */ + public VulnStatus getVulnStatus(String vulnId) { + if (vulnerabilitiesStatus.containsKey(vulnId)) { + return vulnerabilitiesStatus.get(vulnId); + } else return null; + } + + /** + *

addFailure.

+ * + * @param vulnId a {@link java.lang.String} object + * @param e a {@link java.lang.Exception} object + */ + public void addFailure(String vulnId, Exception e) { + failures.put(vulnId, e); + } + + /** + *

lockRepo.

+ * + * @param repo a {@link java.lang.String} object + */ + public void lockRepo(String repo) { + if (!repoLocks.containsKey(repo)) { + repoLocks.put(repo, new ReentrantLock()); + } + repoLocks.get(repo).lock(); + } + + /** + *

unlockRepo.

+ * + * @param repo a {@link java.lang.String} object + */ + public void unlockRepo(String repo) { + if (!repoLocks.containsKey(repo)) { + return; + } + repoLocks.get(repo).unlock(); + } + + /** + * Returns true if an import is underway, false otherwise. + * + * @return a boolean + */ + public boolean isRunningStart() { + return this.stopWatch != null && this.stopWatch.isRunning(); + } + + /** + * Calls kaybee and starts the import of all statements in the data folder. + * + * @param mapCommandOptionValues a {@link java.util.HashMap} object + */ + public synchronized void start(HashMap mapCommandOptionValues) { + + newVulnerabilities = new LinkedHashSet(); + + this.stopWatch = new StopWatch("Import vulnerabilities").start(); + + try { + kaybeeUpdate(); + this.stopWatch.lap("Updated kaybee", true); + + kaybeePull(); + this.stopWatch.lap("Ran kaybee pull", true); + + // Normally, we would call 'kaybee merge", but since this functionality + // has not been implemented yet, we simply copy the folder where the statements + // have been pulled to to another folder + FileUtil.copy(Paths.get(KAYBEE_STMTS_PATH), Paths.get(IMPORT_STMTS_PATH).getParent(), Paths.get(IMPORT_STMTS_PATH).getFileName(), StandardCopyOption.REPLACE_EXISTING); + this.stopWatch.lap("Copied statements", true); + + setUploadConfiguration(mapCommandOptionValues); + + List vulnIds = this.identifyVulnerabilitiesToImport(IMPORT_STMTS_PATH); + startList(IMPORT_STMTS_PATH, mapCommandOptionValues, vulnIds); + retryFailed(IMPORT_STMTS_PATH, mapCommandOptionValues); + + this.stopWatch.stop(); + } catch (Exception e) { + this.stopWatch.stop(e); + } + } + + /** + * Keeps retrying vulnerabilities that failed due to the high amount of requests. + * + * @param statementsPath a {@link java.lang.String} object + * @param mapCommandOptionValues a {@link java.util.HashMap} object + */ + public void retryFailed(String statementsPath, HashMap mapCommandOptionValues) { + + List failuresToRetry = new ArrayList(); + while (true) { + for (String vulnId : failures.keySet()) { + if (failures.get(vulnId) instanceof BackendConnectionException + || failures.get(vulnId) instanceof HttpHostConnectException) { + failuresToRetry.add(vulnId); + } + } + if (failuresToRetry.isEmpty()) { + break; + } else { + log.info( + "Retrying [" + Integer.toString(failuresToRetry.size()) + "] failed vulnerabilities..."); + startList(statementsPath, mapCommandOptionValues, failuresToRetry); + } + } + } + + /** + * Searches in the given folder for directories containing a + *

statements.yaml

file, which correspond to vulnerabilities being + * imported later on. + * + * @param statementsPath a {@link java.lang.String} object + * @return a {@link java.util.List} object + */ + public List identifyVulnerabilitiesToImport(String statementsPath) { + final DirWithFileSearch search = new DirWithFileSearch("statement.yaml"); + Set vulnDirsPaths = search.search(Paths.get(statementsPath)); + List vulnIds = new ArrayList(); + for (Path dirPath : vulnDirsPaths) { + String vulnId = dirPath.getFileName().toString(); + log.debug("Found directory [" + dirPath + "] for vulnerability [" + vulnId + "]"); + setVulnStatus(vulnId, VulnStatus.NOT_STARTED); + vulnIds.add(vulnId); + } + return vulnIds; + } + + /** + * Creates an {@link ImportCommand} for every vulnerability comprised in the + * given list. Depending on the presence of {@link ImportCommand.SEQUENTIAL} + * in the keys of the given arguments, those import commands will be executed + * sequentially are given to the thread pool executor + * {@link Manager#executor}. + * + * @param statementsPath a {@link java.lang.String} refering to the parent + * folder of the vulnerability folders to import + * @param mapCommandOptionValues a {@link java.util.HashMap} with arguments, + * incl. {@link ImportCommand.SEQUENTIAL} + * @param vulnIds a {@link java.util.List} of vulnerabilities to be imported + */ + public synchronized void startList( + String statementsPath, HashMap mapCommandOptionValues, List vulnIds) { + + if (this.executor.isShutdown() || this.executor.isTerminated()) { + this.createNewExecutor(); + } + + failures = new HashMap(); + + // Loop vulnerabilities + for (String vulnId : vulnIds) { + Path vulnDirPath = Paths.get(statementsPath, vulnId); + + // Copy the arguments to avoid concurrent modification + HashMap args = new HashMap(mapCommandOptionValues); + args.put(ImportCommand.DIRECTORY_OPTION, vulnDirPath.toString()); + + // Create the import command. Start right away or submit to executor. + ImportCommand command = new ImportCommand(this, args); + if (mapCommandOptionValues.containsKey(ImportCommand.SEQUENTIAL)) { + command.run(); + } else { + executor.submit(command); + } + } + + // Don't accept new vulns and wait for termination + try { + executor.shutdown(); + executor.awaitTermination(24, TimeUnit.HOURS); + } catch (InterruptedException e) { + log.error("Process interrupted: " + e.getMessage()); + } + } + + private void setUploadConfiguration(HashMap args) { + Object uploadConstruct = args.get(ImportCommand.UPLOAD_CONSTRUCT_OPTION); + VulasConfiguration.getGlobal() + .setProperty( + CoreConfiguration.BACKEND_CONNECT, + (uploadConstruct != null + ? CoreConfiguration.ConnectType.READ_WRITE.toString() + : CoreConfiguration.ConnectType.READ_ONLY.toString())); + } + + /** + * Runs 'kaybe update --force' to update the kaybee binary. + * + * @throws java.io.IOException if any. + * @throws java.lang.InterruptedException if any. + */ + public void kaybeeUpdate() throws ProcessWrapperException, InterruptedException { + ProcessWrapper pw = new ProcessWrapper().setCommand(Paths.get(kaybeeBinaryPath), "update", "--force").setPath(this.tmpDir); + Thread t = new Thread(pw, "kaybee-update"); + t.start(); + t.join(); + } + + /** + * Runs 'kaybee pull -c ' to pull statements from the configured source repositories. + * + * @throws java.io.IOException if any. + * @throws java.lang.InterruptedException if any. + */ + public void kaybeePull() throws Exception { + ProcessWrapper pw = new ProcessWrapper().setCommand(Paths.get(kaybeeBinaryPath), "pull", "-c", kaybeeConfPath).setPath(this.tmpDir); + Thread t = new Thread(pw, "kaybee-pull"); + t.start(); + t.join(); + } + + /** + * Stops all import threads and waits for their termination. + */ + public void stop() { + try { + log.info("Stopping manager..."); + executor.shutdownNow(); + executor.awaitTermination(24, TimeUnit.HOURS); + } catch (InterruptedException e) { + log.error("Process interrupted: " + e.getMessage(), e); + } + } + + /** + * Imports a single vulnerability whose statement.yaml is expected to be in + * the correct folder. + * + * @param vulnDirStr a {@link java.lang.String} object + * @param mapCommandOptionValues a {@link java.util.HashMap} object + * @param vulnId a {@link java.lang.String} object + */ + public void importSingleVuln(HashMap mapCommandOptionValues, String vulnId) { + String dir = IMPORT_STMTS_PATH + File.separator + vulnId; + log.info("Importing vulnerability [" + vulnId + "] from directory [" + dir + "]..."); + + // It is necessary to copy the arguments to avoid concurrent modification + HashMap args = new HashMap(mapCommandOptionValues); + args.put(ImportCommand.DIRECTORY_OPTION, dir); + + ImportCommand command = new ImportCommand(this, args); + command.run(); + } + + /** + *

status.

+ * + * @return a {@link java.lang.String} object + */ + public String status() { + HashMap statusMap = new HashMap(); + + // Counter for each status + HashMap statusCount = new HashMap(); + for (String vul: vulnerabilitiesStatus.keySet()) { + if (!statusCount.containsKey(vulnerabilitiesStatus.get(vul))) { + statusCount.put(vulnerabilitiesStatus.get(vul), 0); + } + statusCount.put(vulnerabilitiesStatus.get(vul), statusCount.get(vulnerabilitiesStatus.get(vul)) + 1); + } + statusMap.put("count", statusCount); + + // New vulns + HashMap newVulnStatus = new HashMap(); + for (String vulnId : newVulnerabilities) { + newVulnStatus.put(vulnId, vulnerabilitiesStatus.get(vulnId)); + } + statusMap.put("new_vulnerabilities", newVulnerabilities); + + // Failures + HashMap failureReasons = new HashMap(); + for (String vulnId : failures.keySet()) { + failureReasons.put(vulnId, failures.get(vulnId).toString()); + } + statusMap.put("failures", failureReasons); + + // Return JSON + return new Gson().toJson(statusMap); + } +} diff --git a/kb-importer/src/main/java/org/eclipse/steady/kb/command/Command.java b/kb-importer/src/main/java/org/eclipse/steady/kb/command/Command.java deleted file mode 100755 index 754d51931..000000000 --- a/kb-importer/src/main/java/org/eclipse/steady/kb/command/Command.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * This file is part of Eclipse Steady. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors - */ -package org.eclipse.steady.kb.command; - -import java.util.HashMap; - -import org.apache.commons.cli.Options; -import org.eclipse.steady.kb.exception.ValidationException; - -/** - * Command Interface - */ -public interface Command { - - enum NAME { - HELP, - VERSION, - IMPORT; - } - - /** - * get the command name - * - * @return a {@link java.lang.String} - */ - Command.NAME getCommandName(); - - /** - * run a command. logic to execute a command with arguments - * - * @param args a {@link java.util.Map}} - */ - void run(HashMap args); - - // TODO: may be we might have to change this to our own bean like - // List rather than the apache-cli Options - /** - * get command options - * - * @return command options - */ - Options getOptions(); - - /** - * validate command with command arguments. Throw a validation exception on any validation error - * - * @param args a {@link java.util.Map} - * @throws org.eclipse.steady.kb.exception.ValidationException if any. - */ - void validate(HashMap args) throws ValidationException; -} diff --git a/kb-importer/src/main/java/org/eclipse/steady/kb/command/CommandExecutor.java b/kb-importer/src/main/java/org/eclipse/steady/kb/command/CommandExecutor.java deleted file mode 100755 index edc3103f5..000000000 --- a/kb-importer/src/main/java/org/eclipse/steady/kb/command/CommandExecutor.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * This file is part of Eclipse Steady. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors - */ -package org.eclipse.steady.kb.command; - -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.ServiceLoader; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.logging.log4j.Logger; -import org.eclipse.steady.kb.exception.CommandLineParserException; -import org.eclipse.steady.kb.exception.ValidationException; - -/** - * command executor - */ -public class CommandExecutor { - - private static final String DIRECTORY_OPTION = "d"; - private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(); - private Map commands = new HashMap<>(); - private static CommandExecutor commandExecutor; - - private CommandExecutor() { - init(); - } - - /** - *

getInstance.

- * - * @return a {@link org.eclipse.steady.kb.command.CommandExecutor} object - */ - public static synchronized CommandExecutor getInstance() { - if (commandExecutor == null) { - commandExecutor = new CommandExecutor(); - } - return commandExecutor; - } - - private void init() { - ServiceLoader serviceProviders = ServiceLoader.load(Command.class); - Iterator iterator = serviceProviders.iterator(); - while (iterator.hasNext()) { - Command command = iterator.next(); - commands.put(command.getCommandName(), command); - } - } - - /** - * command executor - * - * @param _args a array of {java.lang.String} - */ - public void execute(String _args[]) { - if (_args.length == 0) { - log.error("No arguments passed"); - return; - } - - Command command = null; - try { - command = commands.get(Command.NAME.valueOf(_args[0].toUpperCase())); - } catch (IllegalArgumentException e) { - // skip when an unknown command name is passed. Default to import - } - - if (command == null) { - command = new Import(); - } - - Options commandOptions = command.getOptions(); - - HashMap mapCommandOptionValues; - try { - mapCommandOptionValues = CommandParser.parse(_args, commandOptions); - Object rootDirObj = mapCommandOptionValues.get(DIRECTORY_OPTION); - if (rootDirObj != null) { - String rootDir = (String) rootDirObj; - rootDir = getAbsolutePath(rootDir); - mapCommandOptionValues.put(DIRECTORY_OPTION, rootDir); - } - } catch (CommandLineParserException e) { - log.error(e.getMessage()); - printHelp(commandOptions); - return; - } - - try { - command.validate(mapCommandOptionValues); - } catch (ValidationException e) { - log.error(e.getMessage()); - return; - } - - command.run(mapCommandOptionValues); - } - - /** - * Print command help - * - * @param a {@link org.apache.commons.cli.Options} - */ - private void printHelp(Options commandOptions) { - // Showing import help - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("java -jar ", new Import().getOptions()); - } - - /** - * get directory absolute path if it is relative - * - * @param rootDir - * @return absolute path - */ - private String getAbsolutePath(String rootDir) { - return Paths.get(rootDir).toAbsolutePath().normalize().toString(); - } -} diff --git a/kb-importer/src/main/java/org/eclipse/steady/kb/command/CommandParser.java b/kb-importer/src/main/java/org/eclipse/steady/kb/command/CommandParser.java deleted file mode 100755 index 8da0a0ea2..000000000 --- a/kb-importer/src/main/java/org/eclipse/steady/kb/command/CommandParser.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * This file is part of Eclipse Steady. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors - */ -package org.eclipse.steady.kb.command; - -import java.util.Collection; -import java.util.HashMap; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.eclipse.steady.kb.exception.CommandLineParserException; - -/** - * command parser - */ -public class CommandParser { - /** - * parse a command - * - * @param _args array of {@link java.lang.String} - * @param options a {@link org.apache.commons.cli.Options} object - * @throws org.eclipse.steady.kb.exception.CommandLineParserException if any. - * @return a {@link java.util.HashMap} object - */ - public static HashMap parse(String[] _args, Options options) - throws CommandLineParserException { - final CommandLineParser parser = new DefaultParser(); - CommandLine cmd = null; - try { - cmd = parser.parse(options, _args, true); - } catch (ParseException e) { - throw new CommandLineParserException(e.getMessage()); - } - - HashMap mapOptionValues = new HashMap<>(); - Collection