From cdf39e60a620467e77dc9945fb7db572ed3451ba Mon Sep 17 00:00:00 2001 From: Karan Batavia <118820668+karan-batavia@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:50:02 +0530 Subject: [PATCH 1/4] change slack action to upload file (#342) --- .github/workflows/comparison-result.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 2e89d18f..76d8a1e5 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -75,16 +75,16 @@ jobs: echo "MESSAGE<> $GITHUB_ENV echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - + - name: Post results to slack - uses: MeilCli/slack-upload-file@v3 + uses: adrey/slack-file-upload-action@master with: thread_ts: ${{needs.start_workflow.outputs.init_message_ts}} - slack_token: ${{ secrets.SLACK_TOKEN }} - channel_id: ${{ secrets.SLACK_CHANNEL_ID }} - file_path: "/home/runner/work/privado/privado/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip" + channel: ${{ secrets.SLACK_CHANNEL_ID }} # check + path: "/home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip" initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" - file_type: "zip" + filetype: "zip" + token: ${{ secrets.SLACK_TOKEN }} - name: Export workflow output run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt From d7241eefd3c020bb61e94eb20427338f0d0230a3 Mon Sep 17 00:00:00 2001 From: KhemrajSingh Rathore Date: Tue, 7 Nov 2023 20:06:59 +0530 Subject: [PATCH 2/4] Kotlin poc (#344) * kotlin logging rules * api rules * add - kotlin rule * modified mongo rule * add - more rules --- config/systemConfig/kotlin.yaml | 12 +++++++ rules/sinks/leakages/logs/kotlin.yaml | 31 +++++++++++++++++++ rules/sinks/storages/couchdb/java.yaml | 4 +-- rules/sinks/storages/firebase/kotlin.yaml | 24 ++++++++++++++ rules/sinks/storages/mongodb/java.yaml | 4 +-- rules/sinks/storages/realm/kotlin.yaml | 16 ++++++++++ .../third_parties/sdk/google/kotlin.yaml | 10 ++++++ 7 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 config/systemConfig/kotlin.yaml create mode 100644 rules/sinks/leakages/logs/kotlin.yaml create mode 100644 rules/sinks/storages/firebase/kotlin.yaml create mode 100644 rules/sinks/storages/realm/kotlin.yaml create mode 100644 rules/sinks/third_parties/sdk/google/kotlin.yaml diff --git a/config/systemConfig/kotlin.yaml b/config/systemConfig/kotlin.yaml new file mode 100644 index 00000000..f19b2e10 --- /dev/null +++ b/config/systemConfig/kotlin.yaml @@ -0,0 +1,12 @@ +systemConfig: + - key: apiHttpLibraries + value: ^(?i)(org.apache.http|okhttp|org.glassfish.jersey|com.mashape.unirest|java.net.http|java.net.URL|org.springframework.(web|core.io)|groovyx.net.http|org.asynchttpclient|kong.unirest.java|org.concordion.cubano.driver.http|javax.net.ssl|javax.xml.soap|org.apache.axis2|com.sun.xml.messaging.saaj|org.springframework.ws.client|com.eviware.soapui|org.apache.cxf|org.jboss.ws|com.ibm.websphere.sca.extensions.soap|com.sun.xml.ws|org.apache.camel.component.cxf|org.codehaus.xfire|org.apache.synapse|org.apache.wink.client|com.oracle.webservices.internal.api.databinding.Databinding|com.sap.engine.interfaces.webservices.runtime.client).* + + - key: ignoredSinks + value: (?i).*(?<=map|list|jsonobject|json|array|arrays|jsonnode|objectmapper|objectnode).*(put:|get:).* + + - key: apiSinks + value: (?i)(?:url|client|openConnection|request|execute|newCall|load|host|access|fetch|get|getInputStream|getApod|getForObject|getForEntity|list|set|put|post|proceed|trace|patch|Path|send|sendAsync|remove|delete|write|read|assignment|provider|exchange|postForEntity|call|createCall|createEndpoint|dispatch|invoke|newMessage|getInput|getOutput|getResponse|marshall|unmarshall|send|asyncSend) + + - key: apiIdentifier + value: (?i).*((hook|base|auth|prov|endp|install|request|service|gateway|route|resource)(.){0,12}url|(slack|web)(.){0,4}hook|(rest|api|request|service)(.){0,4}(endpoint|gateway|route)).* \ No newline at end of file diff --git a/rules/sinks/leakages/logs/kotlin.yaml b/rules/sinks/leakages/logs/kotlin.yaml new file mode 100644 index 00000000..a87a8549 --- /dev/null +++ b/rules/sinks/leakages/logs/kotlin.yaml @@ -0,0 +1,31 @@ +sinks: + + - id: Leakages.Log.Error + name: Log Error + patterns: + - "(?i)(?:io.github.microutils.kotlin-logging.Logger).*(?:wtf|error|severe|fatal|[.]e[:]).*" + tags: + + - id: Leakages.Log.Warn + name: Log Warn + patterns: + - "(?i)(?:io.github.microutils.kotlin-logging.Logger).*(warn|warning|[.]w[:]).*" + tags: + + - id: Leakages.Log.Debug + name: Log Debug + patterns: + - "(?i)(?:io.github.microutils.kotlin-logging.Logger).*(debug|trace|[.](log|d|v|t)[:]).*" + tags: + + - id: Leakages.Log.Info + name: Log Info + patterns: + - "(?i)(?:io.github.microutils.kotlin-logging.Logger).*(info|[.]i[:]).*" + tags: + + - id: Leakages.Log.Console + name: Log Console + patterns: + - "(?i)(kotlin.io).(?:print|println|write)[:].*" + tags: diff --git a/rules/sinks/storages/couchdb/java.yaml b/rules/sinks/storages/couchdb/java.yaml index 8aded2e0..c0e26f6d 100644 --- a/rules/sinks/storages/couchdb/java.yaml +++ b/rules/sinks/storages/couchdb/java.yaml @@ -9,7 +9,7 @@ sinks: - couchdb.apache.org - apache.org patterns: - - "(?i).*(?:com[.]fourspaces[.]couchdb|org[.]ektorp[.].*couchdb|org[.]lightcouch[.]couchdb|org.*jnosql.*couch.*|com[.]couchbase[.]client).*(?:get|list|head|select|view|find|search|match).*" + - "(?i).*(?:com[.]fourspaces[.]couchdb|org[.]ektorp[.].*couchdb|org[.]lightcouch[.]couchdb|org.*jnosql.*couch.*|com[.]couchbase[.]client|com[.]couchbase[.]lite).*(?:get|list|head|select|view|find|search|match).*" tags: - id: Storages.ApacheCouchDb.Write @@ -18,5 +18,5 @@ sinks: - couchdb.apache.org - apache.org patterns: - - "(?i).*(?:com[.]fourspaces[.]couchdb|.*org[.]ektorp[.].*couchdb|.*org[.]lightcouch[.]CouchDb|org.*jnosql.*couch.*|com[.]couchbase[.]client).*(?:add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|rename|save|set|update|bulkWrite).*" + - "(?i).*(?:com[.]fourspaces[.]couchdb|.*org[.]ektorp[.].*couchdb|.*org[.]lightcouch[.]CouchDb|org.*jnosql.*couch.*|com[.]couchbase[.]clientcom[.]couchbase[.]lite).*(?:add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|rename|save|set|update|bulkWrite).*" tags: diff --git a/rules/sinks/storages/firebase/kotlin.yaml b/rules/sinks/storages/firebase/kotlin.yaml new file mode 100644 index 00000000..2f52666b --- /dev/null +++ b/rules/sinks/storages/firebase/kotlin.yaml @@ -0,0 +1,24 @@ +sinks: + - id: Storages.Firebase.Read + name: Firebase + domains: + - "firebase.google.com" + patterns: + - "(?i)(com[.]google[.]firebase)[.](database).*(?:find|get|select|search|load).*" + - "(?i)(com[.]firebase)[.](database).*(?:find|get|select|search|load).*" + - "(?i)(io[.]firebase)[.](database).*(?:find|get|select|search|load).*" + tags: + + - id: Storages.Firebase.Write + name: Firebase + domains: + - "firebase.google.com" + patterns: + - "(?i)(com[.]google[.]firebase)[.](database).*(?:set|save|delete|insert|update).*" + - "(?i)(com[.]firebase)[.](database).*(?:set|save|delete|insert|update).*" + - "(?i)(io[.]firebase)[.](database).*(?:set|save|delete|insert|update).*" + tags: + + + + \ No newline at end of file diff --git a/rules/sinks/storages/mongodb/java.yaml b/rules/sinks/storages/mongodb/java.yaml index 2d264d1d..8135a93e 100644 --- a/rules/sinks/storages/mongodb/java.yaml +++ b/rules/sinks/storages/mongodb/java.yaml @@ -42,7 +42,7 @@ sinks: domains: - mongodb.com patterns: - - "(?i)(org.springframework.data.mongodb.core.ReactiveMongoTemplate)[.]((find|get|getByKey|getByKeys|getCount|createAggregation|createQuery|queryByExample|exists).*)" + - "(?i)(org.springframework.data.mongodb.core).*[.]((find|get|getByKey|getByKeys|getCount|createAggregation|createQuery|queryByExample|exists).*)" tags: - id: Storages.MongoDB.SpringRepository.Write @@ -50,5 +50,5 @@ sinks: domains: - mongodb.com patterns: - - "(?i)(org.springframework.data.mongodb.core.ReactiveMongoTemplate)[.]((save|update|updateFirst|createUpdateOperations|delete|findAnd(Delete|Modify)).*)" + - "(?i)(org.springframework.data.mongodb.core).*[.]((save|update|updateFirst|createUpdateOperations|delete|findAnd(Delete|Modify)).*)" tags: diff --git a/rules/sinks/storages/realm/kotlin.yaml b/rules/sinks/storages/realm/kotlin.yaml new file mode 100644 index 00000000..af1f1ec7 --- /dev/null +++ b/rules/sinks/storages/realm/kotlin.yaml @@ -0,0 +1,16 @@ +sinks: + - id: Storages.Realm.Read + name: Realm(Read) + domains: + - realm.io + patterns: + - (?i)(io.realm.Realm)[.].*(?:find|get|select|search|load)(.*) + tags: + + - id: Storages.Realm.Write + name: Realm(Write) + domains: + - realm.io + patterns: + - (?i)(io.realm.Realm)[.].*(?:save|delete|insert|update)(.*) + tags: \ No newline at end of file diff --git a/rules/sinks/third_parties/sdk/google/kotlin.yaml b/rules/sinks/third_parties/sdk/google/kotlin.yaml new file mode 100644 index 00000000..8373c67a --- /dev/null +++ b/rules/sinks/third_parties/sdk/google/kotlin.yaml @@ -0,0 +1,10 @@ +sinks: + - id: ThirdParties.SDK.Google.Firebase + name: Firebase + domains: + - "firebase.google.com" + patterns: + - "(?i)(com[.]google[.]firebase)[.](?!(database)).*" + - "(?i)(com[.]firebase)[.](?!(database)).*" + - "(?i)(io[.]firebase)[.](?!(database)).*" + tags: From a37f01b504cf496baf659b45f1605173814c83ca Mon Sep 17 00:00:00 2001 From: Ankit Kumar <118803988+ankit-privado@users.noreply.github.com> Date: Tue, 7 Nov 2023 20:08:33 +0530 Subject: [PATCH 3/4] Added go comparison report support (#345) --- .github/workflows/comparison-result.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 76d8a1e5..8b029636 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -29,7 +29,7 @@ jobs: needs: start_workflow strategy: matrix: - language: ['java', 'python', 'js', 'ruby-1', 'ruby-2'] + language: ['java', 'python', 'js', 'ruby-1', 'ruby-2', 'go'] continue-on-error: true runs-on: ubuntu-latest steps: From 8737e906f7e239e2569097094c2cc060f86e3fd2 Mon Sep 17 00:00:00 2001 From: Karan Batavia <118820668+karan-batavia@users.noreply.github.com> Date: Tue, 7 Nov 2023 20:09:12 +0530 Subject: [PATCH 4/4] fix path in comparison results (#346) --- .github/workflows/comparison-result.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comparison-result.yml b/.github/workflows/comparison-result.yml index 8b029636..112a48af 100644 --- a/.github/workflows/comparison-result.yml +++ b/.github/workflows/comparison-result.yml @@ -81,7 +81,7 @@ jobs: with: thread_ts: ${{needs.start_workflow.outputs.init_message_ts}} channel: ${{ secrets.SLACK_CHANNEL_ID }} # check - path: "/home/runner/work/privado-core/privado-core/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip" + path: "/home/runner/work/privado/privado/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip" initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" filetype: "zip" token: ${{ secrets.SLACK_TOKEN }}