Merge pull request #264 from Privado-Inc/dev

Dev

.github/workflows/comparison-result.yml

@@ -34,7 +34,7 @@ jobs:
           ref: main 
 
       - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}}
-        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -guf -urc
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc
 
       - name: Run aws-export
         run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}}

config/systemConfig/javascript.yaml

@@ -1,12 +1,12 @@
 systemConfig:
   - key: apiHttpLibraries
-    value: (?i)(request|fetch|axios|vue-axios|urllib|http|client|react-query|socketio|xmlhttprequest|node.http|cors|got|apollo|superagent|wretch|@angular\\/common\\/http|.*(HttpClient)).*
+    value: (?i)(request|fetch|axios|vue-axios|urllib|http|client|react-query|socket(.){0,1}io|xmlhttprequest|node.http|cors|got|apollo|superagent|wretch|@angular\\/common\\/http|.*(HttpClient)).*
 
   - key: ignoredSinks
     value: (?i).*(?<=map|list|jsonobject|json|array|arrays|jsonnode|objectmapper|objectnode).*(put:|get:).*
 
   - key: apiSinks
-    value: (?i)(?:url|client|openConnection|request|execute|newCall|load|host|access|usequery|fetch|axios|cors|get|getInputStream|getApod|getForObject|getForEntity|list|set|put|post|proceed|trace|patch|Path|send|sendAsync|remove|delete|write|read|assignment|provider|exchange|postForEntity|call|createCall|createEndpoint|dispatch|invoke|newMessage|getInput|getOutput|getResponse|marshall|unmarshall|send|asyncSend)
+    value: (?i)(?:url|client|openConnection|request|execute|newCall|load|host|access|usequery|fetch|axios|cors|get|getInputStream|getApod|getForObject|getForEntity|list|set|put|post|proceed|trace|patch|Path|send|sendAsync|remove|delete|write|read|assignment|provider|exchange|postForEntity|call|createCall|createEndpoint|dispatch|invoke|newMessage|getInput|getOutput|getResponse|marshall|unmarshall|send|asyncSend|emit|on)
 
   - key: apiIdentifier
     value: (?i).*((hook|base|auth|prov|endp|install|cloud|host|request|service|gateway|route|resource|upload|api|worker)(.){0,12}url|(slack|web)(.){0,4}hook|(rest|api|host|cloud|request|service)(.){0,4}(endpoint|gateway|route)).*

rules/sinks/storages/cookiemanager/javascript.yaml

@@ -1,19 +1,45 @@
 sinks:
-  - id: Storages.Web.Cookie
-    name: Web Storage Cookie
+
+  - id: Storages.Web.Cookie.Write
+    name: Web Storage Cookie(Write)
     patterns:
-      - "(?i).*(getCookie|setCookie|deleteCookie|removeCookie|useCookies)"
+      - "(?i)(.*cookie.*set.*)|(.*setCookie)"
+    tags:
+
+  - id: Storages.Web.Cookie.Delete
+    name: Web Storage Cookie(Delete)
+    patterns:
+      - "(?i).*(deleteCookie|removeCookie)"
     tags:
       law: GDPR
 
-  - id: Storages.Web.LocalStorage
-    name: Web LocalStorage
+  - id: Storages.Web.LocalStorage.Write
+    name: Web LocalStorage(Write)
+    patterns:
+      - "(?i)(localStorage).*(setItem|clear|removeItem)"
+    tags:
+
+  - id: Storages.Web.SessionStorage.Write
+    name: Web SessionStorage(Write)
     patterns:
-      - "(?i)(localStorage).*(setItem|clear|removeItem|getItem)"
+      - "(?i)(\\bstorage\\b|sessionstorage)(.*)(setItem|clear|removeItem)"
     tags:
 
-  - id: Storages.Web.SessionStorage
-    name: Web SessionStorage
+  - id: Storages.Web.Cookie.READ
+    name: Web Storage Cookie(READ)
     patterns:
-      - "(?i)(\\bstorage\\b|sessionstorage)(.*)(setItem|clear|removeItem|getItem)"
+      - "(?i).*(getCookie|useCookies)"
     tags:
+      law: GDPR
+
+  - id: Storages.Web.LocalStorage.READ
+    name: Web LocalStorage(READ)
+    patterns:
+      - "(?i)(localStorage).*(clear|getItem)"
+    tags:
+
+  - id: Storages.Web.SessionStorage.READ
+    name: Web SessionStorage(READ)
+    patterns:
+      - "(?i)(\\bstorage\\b|sessionstorage)(.*)(getItem)"
+    tags:

rules/sinks/storages/messagingqueue/java.yaml

@@ -13,7 +13,7 @@ sinks:
     domains:
       - apache.com
     patterns:
-      - "(?i)((org[.]springframework[.]kafka[.]core[.]KafkaConsumer[.](receive|poll|subscribe)[:])|(org[.]apache[.]kafka[.]clients[.]consumer[.]Consumer[.](resume|commitAsync))|(org[.]apache[.]kafka[.]clients[.]consumer[.]((ConsumerRecord[.](partition|topic|value))|(KafkaConsumer[.](poll|receive|subscribe|unsubscribe|assign|commit|seek|pause|resume|close|commitAsync)[:])))|(org[.]springframework[.]kafka[.]listener[.]MessageListener[.]onMessage)|(org[.]springframework[.]kafka[.]listener[.]KafkaMessageListenerContainer[.](start|stop|pause|resume))|(com[.]salesforce[.]kafka[.]client[.]consumer[.]KafkaConsumer[.](subscribe|unsubscribe|assign|seek|poll))).*"
+      - "(?i)((org[.]springframework[.]kafka[.]core[.]KafkaConsumer.*(receive|poll|subscribe)[:])|(org[.]apache[.]kafka[.]clients[.]consumer[.]Consumer.*(resume|commitAsync))|(org[.]apache[.]kafka[.]clients[.]consumer[.]((ConsumerRecord.*(partition|topic|value))|((KafkaConsumer|Consumer).*(poll|receive|subscribe|unsubscribe|assign|commit|seek|pause|resume|close|commitAsync)[:])))|(org[.]springframework[.]kafka[.]listener[.]MessageListener[.]onMessage)|(org[.]springframework[.]kafka[.]listener[.]KafkaMessageListenerContainer[.](start|stop|pause|resume))|(com[.]salesforce[.]kafka[.]client[.]consumer[.]KafkaConsumer.*(subscribe|unsubscribe|assign|seek|poll))).*"
     tags:
 
   - id: Messaging.Queue.AMQP.Rabbit.Producer
@@ -29,11 +29,11 @@ sinks:
     domains:
       - springframework.org
     patterns: 
-      - "(?i)(org[.]springframework[.]jms[.]core[.]JmsTemplate[.](doSend|send|sendAndReceive|convertAndSend|execute)[:]|(jakarta[.]jms[.]JMSProducer|javax[.]jms[.]MessageProducer)[.](send)[:]).*"
-      - "(?i)org.apache.qpid.jms.JmsMessageProducer.send[:].*"
-      - "(?i)com.ibm.mq.jms.MQQueueSender.send[:].*"
-      - "(?i)org.apache.camel.component.jms.JmsProducer.(process|onExchange)[:].*"
-      - "(?i)org.apache.nifi.jms.processors.JMSProducer.sendMessage[:].*"
+      - "(?i)(org[.]springframework[.]jms[.]core[.]JmsTemplate.*(doSend|send|sendAndReceive|convertAndSend|execute)[:]|(jakarta[.]jms[.]JMSProducer|javax[.]jms[.]MessageProducer).*(send)[:]).*"
+      - "(?i)org.apache.qpid.jms.JmsMessageProducer.*send[:].*"
+      - "(?i)com.ibm.mq.jms.MQQueueSender.*send[:].*"
+      - "(?i)org.apache.camel.component.jms.JmsProducer.*(process|onExchange)[:].*"
+      - "(?i)org.apache.nifi.jms.processors.JMSProducer.*sendMessage[:].*"
     tags:
 
   - id: Messaging.Service.AmazonSQS.Producer
@@ -49,19 +49,19 @@ sinks:
     domains:
       - amazonaws.com
     patterns: 
-      - "(?i).*(com[.]amazonaws[.]services[.]sqs[.]AmazonSQS[.](receiveMessage|receiveMessageBatch|deleteMessage|deleteMessageBatch)).*"
+      - "(?i).*(com[.]amazonaws[.]services[.]sqs[.]AmazonSQS.*(receiveMessage|receiveMessageBatch|deleteMessage|deleteMessageBatch)).*"
     tags:
 
   - id: Messaging.Service.JMS.Consumer
     name: JMS (Consumer)
     domains:
       - springframework.org
     patterns: 
-      - "(?i)(org[.]springframework[.]jms[.]core[.]JmsTemplate[.](doReceive|receive)[:]|(jakarta[.]jms[.]JMSConsumer|javax[.]jms[.]MessageConsumer)[.](receive|receiveBody|receiveNoWait|receiveBodyNoWait)[:]).*"
-      - "(?i)org.apache.qpid.jms.JmsMessageProducer.receive[:].*"
-      - "(?i)com.ibm.mq.jms.MQQueueSender.receive[:].*"
-      - "(?i)org.apache.camel.component.jms.JmsConsumer.processMessage[:].*"
-      - "(?i)org.springframework.integration.jms.JmsSendingMessageHandler.onMessage[:].*"
+      - "(?i)(org[.]springframework[.]jms[.]core[.]JmsTemplate.*(doReceive|receive)[:]|(jakarta[.]jms[.]JMSConsumer|javax[.]jms[.]MessageConsumer).*(receive|receiveBody|receiveNoWait|receiveBodyNoWait)[:]).*"
+      - "(?i)org.apache.qpid.jms.JmsMessageProducer.*receive[:].*"
+      - "(?i)com.ibm.mq.jms.MQQueueSender.*receive[:].*"
+      - "(?i)org.apache.camel.component.jms.JmsConsumer.*processMessage[:].*"
+      - "(?i)org.springframework.integration.jms.JmsSendingMessageHandler.*onMessage[:].*"
     tags:
 
   - id: Messaging.Queue.Mosquitto.Producer
@@ -77,5 +77,5 @@ sinks:
     domains:
       - mosquitto.org
     patterns: 
-      - "(?i)org.eclipse.paho.client.mqttv3.MqttClient.(subscribe|messageArrived)[:].*"
+      - "(?i)org.eclipse.paho.client.mqttv3.MqttClient.*(subscribe|messageArrived)[:].*"
     tags:

rules/sinks/storages/mongodb/javascript.yaml

@@ -5,6 +5,21 @@ sinks:
     domains:
       - mongodb.com
     patterns:
-      - "(?i).*(mongoose|MongoClient).*"
-      - "(?:mongodb|mongoose|mongo-|connect-mongo|mquery|mpath|mongojs|winston-mongodb|feathers-mongoose|koa2-ratelimit|gridfs-stream|aedes-persistence-mongodb|mockgoose|mubsub|minimongo|uuid-mongodb|@fastify/mongodb|gridfs-promise|feathers-mongodb-fuzzy-search|rus-diff|recachegoose|baqend|@onehilltech/blueprint-mongodb|cachegoose|@treehouses/cli|gridfs-locking-stream|hapi-mongo-models|forerunnerdb|gridfs|payload|@lenne.tech/nest-server|database-cleaner|yams|@firstteam102/connect-mongo|json2mongo|@oguzbey/mongoose-beautiful-unique-validation|node-mongotools|ascoltatori|@casbin/mongo-changestream-watcher|@appveen/swagger-mongoose-crud|tingodb|generator-ng-fullstack|objectid|opentelemetry-instrumentation-mongoose|@immjunaid/create-express-restapis|apollo-passport-mongodb-driver|graphql-advanced-projection|jsonquery-engine|drop-mongodb-collections|nosqldbm-converter|nedb-lite|promised-mongo|feathers-mongodb|flatten-obj|mongoskin|sift|migrate-mongo|denque|mqemitter-mongodb|to-mongodb-core|graphql-mongodb-projection|jugglingdb|gulp-mongodb-data|thunkify-mongodb|joi-objectid|electron-squirrel-startup|node-express-mongodb-jwt-rest-api-skeleton|@caruuto/api-mongodb|sharedb-mongo|@chrishenderson/mongodb-queue|twitter2mongodb|@lpgroup/feathers-mongodb|@neo9/n9-mongodb-migration|sails-mongo|mongolass|w-orm-mongodb).*"
+      - "(?:mquery|mpath|mongojs|mongodb[.]net|winston-mongodb|feathers-mongoose|koa2-ratelimit|gridfs-stream|aedes-persistence-mongodb|mockgoose|mubsub|minimongo|uuid-mongodb|@fastify/mongodb|gridfs-promise|feathers-mongodb-fuzzy-search|rus-diff|recachegoose|baqend|@onehilltech/blueprint-mongodb|cachegoose|@treehouses/cli|gridfs-locking-stream|hapi-mongo-models|forerunnerdb|gridfs|payload|@lenne.tech/nest-server|database-cleaner|yams|@firstteam102/connect-mongo|json2mongo|@oguzbey/mongoose-beautiful-unique-validation|node-mongotools|ascoltatori|@casbin/mongo-changestream-watcher|@appveen/swagger-mongoose-crud|tingodb|generator-ng-fullstack|objectid|opentelemetry-instrumentation-mongoose|@immjunaid/create-express-restapis|apollo-passport-mongodb-driver|graphql-advanced-projection|jsonquery-engine|drop-mongodb-collections|nosqldbm-converter|nedb-lite|promised-mongo|feathers-mongodb|flatten-obj|mongoskin|sift|migrate-mongo|denque|mqemitter-mongodb|to-mongodb-core|graphql-mongodb-projection|jugglingdb|gulp-mongodb-data|thunkify-mongodb|joi-objectid|electron-squirrel-startup|node-express-mongodb-jwt-rest-api-skeleton|@caruuto/api-mongodb|sharedb-mongo|@chrishenderson/mongodb-queue|twitter2mongodb|@lpgroup/feathers-mongodb|@neo9/n9-mongodb-migration|sails-mongo|mongolass|w-orm-mongodb).*"
     tags:
+
+  - id: Storages.MongoDB.Read
+    name: MongoDB (Read)
+    domains:
+      - mongodb.com
+    patterns:
+      - "(?i)(?:mongodb|mongoose|mongo-|connect-mongo|.*(mongoose|MongoClient|connect-mongodb-session)).*(?:findOne|find|aggregate|command|findOneAndUpdate)"
+    tags:
+
+  - id: Storages.MongoDB.Write
+    name: MongoDB (Write)
+    domains:
+      - mongodb.com
+    patterns:
+      - "(?i)(?:mongodb|mongoose|mongo-|connect-mongo|.*(mongoose|MongoClient|connect-mongodb-session)).*(?:insertOne|insertMany|deleteOne|deleteMany|updateOne|updateMany)"
+    tags:

rules/sinks/storages/redis/java.yaml

@@ -27,6 +27,7 @@ sinks:
     domains:
       - redis.io
     patterns:
+      - (?i)(redis.clients.jedis.*get.*)
       - (?i)(redis.clients.jedis[.](providers.ClusterConnectionProvider|connection)[.])(get)(node[s]?|one|raw)?(:)(.*)
       - (?i)(redis.clients.jedis[.](providers.ClusterConnectionProvider|connection)[.])(get)((objectmulti)?bulk|integer|statuscode)(reply)(:)(.*)
       - (?i)(redis.clients.jedis.CommandObjects[.])(m|h|p)?(get)(all|del)?(:)(.*)
@@ -37,6 +38,7 @@ sinks:
     domains:
       - redis.io
     patterns:
+      - (?i)(redis.clients.jedis.*set.*)
       - (?i)(redis.clients.jedis.CommandObjects.getset:)(.*)
       - (?i)(redis.clients.jedis.CommandObjects[.])(m|h|p)?(set)(one|user|range|nx|ex)?(:)(.*)
     tags:

rules/sinks/storages/redis/javascript.yaml

@@ -1,8 +1,17 @@
 sinks:
   - id: Storages.Redis.Read
-    name: Redis DB
+    name: Redis DB(Read)
     domains:
       - redis.io
     patterns:
-      - (?i)(redis|connect-redis|ioredis|cache-manager|@socket.io\\/redis-adapter|@fastify\\/redis)
+      - (?i)(redis.*(get|hGetAll))
+      - (?i)(connect-redis|ioredis|cache-manager|@socket.io\\/redis-adapter|@fastify\\/redis)
+    tags:
+
+  - id: Storages.Redis.Write
+    name: Redis DB(Write)
+    domains:
+      - redis.io
+    patterns:
+      - (?i)(redis.*(setex|set|hSet|add|sAdd))
     tags: