Merge pull request #14 from noobpk/dev

Bump to version 0.1.4

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 23/09/2023
+- Release v0.1.4 Gemini-self-protector for Flask
+- Add monitor page
+- Fix bug
+
 ## 12/07/2023
 - Release v0.1.2 Gemini-self-protector for Flask
 - Update for work with new predict server

README.md

@@ -17,7 +17,13 @@ By seamlessly integrating RASP into your application's runtime fabric, Gemini-Se
 
 ## Gemini Component
 
-![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/d53ffcd3-2a1a-4931-b050-896161adf085)
+![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/e825d5ef-90d9-419c-b1d5-76406b770b04)
+
+👉 Plugin : [gemini-self-protector](https://github.com/noobpk/gemini-self-protector)
+
+👉 AI : [gemini-predict-serve](https://github.com/noobpk/gemini-predict-serve)
+
+👉 BigData : [gemini-bigdata-serve](https://github.com/noobpk/gemini-bigdata-serve)
 
 ## Gemini Plugin Architecture
 
@@ -69,42 +75,35 @@ Gemini supports 3 modes and recommends sensitivity levels for the application to
 ## Implement prediction module
 💪 You can implement your own prediction module extremely simply and quickly. Details at [gemini-predict-serve](https://github.com/noobpk/gemini-predict-serve)
 
-## GUI Features
-
-👉 Monitor Abnormal Event
-
-👉 Hot Configuration
-
-👉 Access Control List
-
-👉 Log Activity
-
-👉 Dependency Vulnerability Check
-
 ## Demo
 
 [Gemini-Self-Protector | Demo | Install - Configurate - Usage](https://youtu.be/sUJsJE29KcE)
 
 ## Screenshot
 
-### Dashboard Screen
+### Dashboard
 
-![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/112e227b-9f43-4189-b1fd-038e7cd324ee)
+<img width="1440" alt="image" src="https://github.com/noobpk/gemini-self-protector/assets/31820707/068048ef-42cf-4032-b064-137d69abccb6">
 
-### Configurate Screen
+### Monitoring
 
-![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/e2b4fc0b-c188-4c52-b21e-afd7e8d52582)
+![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/c4308492-c283-4c8c-a22f-8e503079b30e)
+
+### Configurate
+
+<img width="1440" alt="image" src="https://github.com/noobpk/gemini-self-protector/assets/31820707/d8e4376f-72d1-4a7d-8a96-838b9436b0b1">
 
 ### Access Control List
 
-![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/ca2ae9a7-7956-4b83-866d-8fa5f9c4ce2b)
+<img width="1440" alt="image" src="https://github.com/noobpk/gemini-self-protector/assets/31820707/496033ec-e953-4ca4-9d16-73a402161f8a">
 
 ### Dependency Check
 
 ![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/e5b58af5-fe2a-4f3a-ab03-e25923bd72ee)
 
 ### Endpoint 
-![image](https://github.com/noobpk/gemini-self-protector/assets/31820707/67db7eed-5c12-452d-89ae-80a88b10817a)
+
+<img width="1440" alt="image" src="https://github.com/noobpk/gemini-self-protector/assets/31820707/109717d9-aac2-4c97-8e36-133e2d6365cb">
 
 ## Contributing
 

gemini-python/gemini_self_protector/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## v0.1.4 (23/09/2023)
+
+- Fix bug 
+- Add monitor page
+
 ## v0.1.2-c (01/08/2023)
 
 - Fix bug 'NoneType' object is not subscriptable in decoder

gemini-python/gemini_self_protector/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gemini_self_protector"
-version = "0.1.3"
+version = "0.1.4"
 description = "Runtime Application Self-Protection"
 authors = ["lethanhphuc"]
 license = "MIT"

gemini-python/gemini_self_protector/src/gemini_self_protector/_config.py

@@ -30,6 +30,7 @@ def __init__(self, working_directory):
                 global_protect_mode='monitor',
                 max_content_length=52428800,
                 sensitive_value=50,
+                is_predict_header = 0,
                 http_method_allow=json.dumps(
                     ['OPTIONS', 'GET', 'POST', 'PUT', 'DELETE']),
                 cors={'origin': '*', 'methods': '*',

gemini-python/gemini_self_protector/src/gemini_self_protector/_gui.py

@@ -85,7 +85,7 @@ def gemini_install():
                             confirm_password = request.form['cpwd']
                             notification_channel = request.form['radio-channel']
                             predict_server_key_auth = request.form['key-auth-server-value']
-                            predict_server = request.form['predit-server-value']
+                            predict_server = request.form['predict-server-value']
                             telegram_token = ''
                             telegram_chat_id = ''
                             notification_webhook = ''
@@ -273,6 +273,20 @@ def gemini_dashboard():
                     logger.error("[x_x] Something went wrong at {0}, please check your error message.\n Message - {1}".format(
                         'nested_service.route.gemini_dashboard', e))
 
+            @nested_service.route('/monitor', methods=['GET'])
+            def gemini_monitor():
+                try:
+                    if not session.get('gemini_logged_in'):
+                        flash('Please login', 'required_login')
+                        return redirect(url_for('nested_service.gemini_login'))
+
+                    gemini_config = _Gemini.get_gemini_config()
+                    return render_template('gemini-protector-gui/home/monitor.html', _socketio=gemini_config.socketio)
+
+                except Exception as e:
+                    logger.error("[x_x] Something went wrong at {0}, please check your error message.\n Message - {1}".format(
+                        'nested_service.route.gemini_monitor', e))
+
             @nested_service.route('/configurate', methods=['GET', 'POST'])
             def gemini_configurate():
                 try:
@@ -292,9 +306,11 @@ def gemini_configurate():
                         protect_response_status = request.form['protect_response_status']
                         acl_status = request.form['acl_status']
                         anti_dos = request.form['anti_dos_status']
+                        predict_header = request.form['predict_header_status']
                         max_request_per_min = request.form['max_request_per_min']
+                        socketio = request.form['socketio']
 
-                        if _Gemini.validator_protect_mode(protect_mode) and _Gemini.validator_sensitive_value(sensitive_value) and max_content_length.isdigit() and _Gemini.validator_http_method(http_method) and _Gemini.validator_on_off_status(safe_redirect_status) and _Gemini.validator_trust_domain(trust_domain_list) and _Gemini.validator_on_off_status(protect_response_status) and _Gemini.validator_on_off_status(acl_status) and _Gemini.validator_on_off_status(anti_dos) and max_request_per_min.isdigit():
+                        if _Gemini.validator_protect_mode(protect_mode) and _Gemini.validator_sensitive_value(sensitive_value) and max_content_length.isdigit() and _Gemini.validator_http_method(http_method) and _Gemini.validator_on_off_status(safe_redirect_status) and _Gemini.validator_trust_domain(trust_domain_list) and _Gemini.validator_on_off_status(protect_response_status) and _Gemini.validator_on_off_status(acl_status) and _Gemini.validator_on_off_status(anti_dos) and _Gemini.validator_on_off_status(predict_header) and  max_request_per_min.isdigit():
 
                             _Gemini.update_gemini_config({
                                 "predict_server": predict_server,
@@ -307,7 +323,9 @@ def gemini_configurate():
                                 "trust_domain": json.dumps(trust_domain_list),
                                 "enable_acl": int(acl_status),
                                 "anti_dos": int(anti_dos),
-                                "max_requests_per_minute": int(max_request_per_min)
+                                "is_predict_header": int(predict_header),
+                                "max_requests_per_minute": int(max_request_per_min),
+                                "socketio": socketio
                             })
                             flash('Configuration update successful',
                                   'config_update_success')
@@ -338,7 +356,9 @@ def gemini_configurate():
                                                _protect_response=gemini_config.protect_response,
                                                _is_enable_acl=gemini_config.enable_acl,
                                                _is_anti_dos=gemini_config.anti_dos,
-                                               _max_request_per_min=gemini_config.max_requests_per_minute
+                                               _is_predict_header=gemini_config.is_predict_header,
+                                               _max_request_per_min=gemini_config.max_requests_per_minute,
+                                               _socketio = gemini_config.socketio
                                                )
                 except Exception as e:
                     logger.error("[x_x] Something went wrong at {0}, please check your error message.\n Message - {1}".format(
@@ -502,6 +522,7 @@ def gemini_get_event():
                             "res_content": str(record.res_content),
                             "attack_type": record.attack_type,
                             "predict": record.predict,
+                            "hash": record.hash,
                             "latitude": record.latitude,
                             "longitude": record.longitude,
                             "created_at": record.created_at,

gemini-python/gemini_self_protector/src/gemini_self_protector/_model.py

@@ -24,6 +24,7 @@ class tb_Config(Base):
     database_path = Column(String)
     cors = Column(JSON)
     predict_server = Column(String)
+    is_predict_header = Column(Integer)
     global_protect_mode = Column(String)
     http_method_allow = Column(String)
     max_content_length = Column(Integer)
@@ -40,6 +41,7 @@ class tb_Config(Base):
     telegram_chat_id = Column(String)
     telegram_token = Column(String)
     trust_domain = Column(String)
+    socketio = Column(String)
     working_directory = Column(String)
     created_at = Column(DateTime, default=func.now())
     updated_at = Column(DateTime, default=func.now(), onupdate=func.now())
@@ -72,6 +74,7 @@ class tb_RequestLog(Base):
     useragent = Column(String)
     attack_type = Column(String)
     predict = Column(Float)
+    hash = Column(String)
     event_id = Column(String)
     latitude = Column(Float)
     longitude = Column(Float)

gemini-python/gemini_self_protector/src/gemini_self_protector/_protect.py

@@ -211,7 +211,12 @@ def __protect_flask_request__(gemini_protect_mode) -> None:
                 _method, _full_path, _headers)
             _request_body = _body
 
-            _predict_request = '{}\r{}'.format(_request_header, _request_body)
+            is_predict_header = _Config.get_tb_config().is_predict_header
+
+            if int(is_predict_header) == 1:
+                _predict_request = '{}\r{}'.format(_request_header, _request_body)
+            else:
+                _predict_request = '{}'.format(_request_body)
 
             _ticket = _Utils.insident_ticket()
 
@@ -243,7 +248,7 @@ def __protect_flask_request__(gemini_protect_mode) -> None:
                 return {"Status": status}
 
         except Exception as e:
-            logger.error("[x_x] Something went wrong, please check your error message.\n Message - {}".format('_Protect.__protect_flask_response__', e))
+            logger.error("[x_x] Something went wrong, please check your error message.\n Message - {}".format('_Protect.__protect_flask_request__', e))
 
     def __protect_flask_response__(safe_redirect, original_response, gemini_protect_mode) -> None:
         try:
@@ -292,4 +297,4 @@ def __protect_flask_response__(safe_redirect, original_response, gemini_protect_
                 return {"Status": status}
 
         except Exception as e:
-            logger.error("[x_x] Something went wrong, please check your error message.\n Message - {}".format('_Protect.__protect_flask_request__', e))
+            logger.error("[x_x] Something went wrong, please check your error message.\n Message - {}".format('_Protect.__protect_flask_response__', e))

gemini-python/gemini_self_protector/src/gemini_self_protector/_utils.py

@@ -196,9 +196,10 @@ def predict_server_health() -> None:
         try:
             predict_server = _Config.get_tb_config().predict_server
             if predict_server:
+                client_ip = _Utils.flask_client_ip()
                 headers = {"Content-Type": "application/json"}
                 response = requests.post(
-                    f'{predict_server}/predict', json={"data": "healthcheck"}, headers=headers)
+                    f'{predict_server}/predict', json={"ip": client_ip, "data": "healthcheck"}, headers=headers)
                 if response and response.status_code == 200:
                     return True
                 else:
@@ -215,10 +216,11 @@ def validate_key_auth(_key) -> None:
         try:
             if _key:
                 predict_server = _Config.get_tb_config().predict_server
+                client_ip = _Utils.flask_client_ip()
                 headers = {"Content-Type": "application/json",
                            "Authorization": _key}
                 response = requests.post(
-                    f'{predict_server}/predict', json={"data": "healthcheck"}, headers=headers)
+                    f'{predict_server}/predict', json={"ip": client_ip, "data": "healthcheck"}, headers=headers)
                 data = response.json()
                 if response.status_code == 200 and 'accuracy' in data:
                     _Config.update_tb_config({
@@ -378,10 +380,11 @@ def validate_trust_domain(trust_domain_list) -> None:
     def validate_predict_server(_server, _key) -> None:
         try:
             if re.match(r'https?://\S+', _server):
+                client_ip = _Utils.flask_client_ip()
                 headers = {"Content-Type": "application/json",
                            "Authorization": _key}
                 response = requests.post(
-                    f'{_server}/predict', json={"data": "healthcheck"}, headers=headers)
+                    f'{_server}/predict', json={"ip": client_ip, "data": "healthcheck"}, headers=headers)
                 data = response.json()
                 if response.status_code == 200 and 'accuracy' in data:
                     return True

gemini-python/gemini_self_protector/src/gemini_self_protector/gemini_self_protector.py

@@ -36,12 +36,17 @@ def _gemini_self_protect(f):
             @wraps(f)
             def __gemini_self_protect(*args, **kwargs):
                 _Gemini.calulate_total_access()
+                is_install = _Gemini.get_gemini_config().isinstall
                 _ip_address = _Gemini.get_flask_client_ip()
                 is_enable_anti_dos = _Gemini.get_gemini_config().anti_dos
                 is_enable_acl = _Gemini.get_gemini_config().enable_acl
                 is_protect_response = _Gemini.get_gemini_config().protect_response
                 mrpm = _Gemini.get_gemini_config().max_requests_per_minute
 
+                if int(is_install) == 0:
+                    response = make_response("Please setup gemini-self-protector.")
+                    return response
+
                 if int(is_enable_anti_dos):
                     request_key = f"{_ip_address}_{datetime.now().strftime('%Y-%m-%d %H:%M')}"
 

gemini-python/gemini_self_protector/src/gemini_self_protector/resource/assets/js/install.js

@@ -193,7 +193,7 @@ jQuery(document).ready(function () {
                 "Authorization": keyAuthValue
             },
             contentType: "application/json",
-            data: JSON.stringify({ data: "healthcheck" }),
+            data: JSON.stringify({ ip: "0.0.0.0", data: "healthcheck" }),
             success: function (response) {
                 // Handle the success response
                 if (response.accuracy) {

gemini-python/gemini_self_protector/src/gemini_self_protector/resource/templates/home/config.html

@@ -118,6 +118,21 @@ <h5>Configurate</h5>
                                                     value="{{_sensitive_value}}" name="sensitive_value">
                                             </div>
                                             <!-- [ Sensitive Value ] stop -->
+                                            <!-- [ Predict Header ] start -->
+                                            <div class="form-group">
+                                                <label>Predict Header</label>
+                                                <div class="custom-control custom-radio custom-control-inline">
+                                                    <input type="radio" id="predict-header-on" name="predict_header_status" value="1" class="custom-control-input"
+                                                        {% if _is_predict_header==1 %}checked{% endif %}>
+                                                    <label class="custom-control-label" for="predict-header-on">On</label>
+                                                </div>
+                                                <div class="custom-control custom-radio custom-control-inline">
+                                                    <input type="radio" id="predict-header-off" name="predict_header_status" value="0" class="custom-control-input"
+                                                        {% if _is_predict_header==0 %}checked{% endif %}>
+                                                    <label class="custom-control-label" for="predict-header-off">Off</label>
+                                                </div>
+                                            </div>
+                                            <!-- [ Predict Header ] stop -->
                                             <!-- [ Content-Lenght ] start -->
                                             <div class="form-group">
                                                 <label>Max Content-Length: 1 * 1024 * 1024 = 1MB</label>
@@ -224,6 +239,12 @@ <h5>Configurate</h5>
                                                     name="trust_domain_list">
                                             </div>
                                             <!-- [ Trust Domain ] stop -->
+                                            <!-- [ Socketio ] start -->
+                                            <div class="form-group">
+                                                <label>Socket.io</label>
+                                                <input type="text" class="form-control" placeholder="http://" value="{{_socketio}}" name="socketio">
+                                            </div>
+                                            <!-- [ Socketio ] stop -->
                                             <button type="submit" class="btn btn-primary">Submit</button>
                                         </form>
                                     </div>

gemini-python/gemini_self_protector/src/gemini_self_protector/resource/templates/home/index.html

@@ -320,6 +320,8 @@ <h5>Event Description</h5>
                                                     <dd class="col-sm-9" id="response_value"></dd>
                                                     <dt class="col-sm-3">Response Content</dt>
                                                     <dd class="col-sm-9" id="res_content_value"></dd>
+                                                    <dt class="col-sm-3">Hash</dt>
+                                                    <dd class="col-sm-9" id="hash"></dd>
                                                 </dl>
                                             </div>
                                         </div>
@@ -468,6 +470,8 @@ <h5>Event Description</h5>
                     $('#req_body_value').text(data.req_body ? data.req_body : 'N/A');
                     $('#response_value').html(data.response ? data.response.replace(/(?:\r\n|\r|\n)/g, '<br>') : 'N/A');
                     $('#res_content_value').text(data.res_content ? data.res_content : 'N/A');
+                    $('#hash').text(data.hash ? data.hash : 'N/A');
+
                 },
                 error: function (error) {
                     console.error('Error:', error);