Add filelock to ensure file access single thread

Manual release file lock

Author: terasum

drls/app.py

@@ -6,6 +6,9 @@
 from drls.extensions import bcrypt, cache, csrf_protect, db, debug_toolbar, login_manager, migrate, webpack
 from drls.settings import ProdConfig
 from drls.utils import JSONR
+from filelock import Timeout, FileLock
+
+import os
 
 
 def create_app(config_object=ProdConfig):
@@ -20,6 +23,7 @@ def create_app(config_object=ProdConfig):
     register_errorhandlers(app)
     register_shellcontext(app)
     register_commands(app)
+    register_user_variable(app)
     return app
 
 
@@ -72,3 +76,19 @@ def register_commands(app):
     app.cli.add_command(commands.lint)
     app.cli.add_command(commands.clean)
     app.cli.add_command(commands.urls)
+
+def register_user_variable(app):
+    app.config.lock_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['LOCK_FILE_NAME'])
+    app.config.seed_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMSEED_FILE_NAME'])
+    app.config.xls_file_path  = os.path.join(app.config['UPLOAD_FOLDER'], app.config['STUDATA_FILE_NAME'])
+    app.config.num_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMNUM_FILE_NAME'])
+
+    app.config.lock_file_lock_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['LOCK_FILE_NAME']+'.lock')
+    app.config.seed_file_lock_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMSEED_FILE_NAME']+'.lock')
+    app.config.xls_file_lock_path  = os.path.join(app.config['UPLOAD_FOLDER'], app.config['STUDATA_FILE_NAME']+'.lock')
+    app.config.num_file_lock_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMNUM_FILE_NAME']+'.lock')
+
+    app.config.num_file_lock = FileLock(app.config.num_file_lock_path)
+    app.config.seed_file_lock = FileLock(app.config.seed_file_lock_path)
+    app.config.xls_file_lock = FileLock(app.config.xls_file_lock_path)
+    app.config.lock_file_lock = FileLock(app.config.lock_file_lock_path)

drls/data/.gitignore

@@ -3,3 +3,4 @@ studata.xls
 random.txt
 num.txt
 lock.txt
+*.lock

drls/data/random_seed.txt

@@ -0,0 +1 @@
+1155123

drls/public/__init__.py

@@ -1,3 +1,4 @@
 # -*- coding: utf-8 -*-
 """The public module, including the homepage and user auth."""
 from . import views  # noqa
+

drls/public/views.py

@@ -6,12 +6,12 @@
 from drls.utils import read_excel, cal_range, file_exists,allowed_file,app_dir,load_md,JSONR
 from drls.rng import RandomGenerator
 from werkzeug.utils import secure_filename
-import hashlib
 
 import os
-import json
 import drls.errcode as ERRCODE
 
+
+
 blueprint = Blueprint('public', __name__, static_folder='../static')
 
 
@@ -22,8 +22,8 @@ def home():
     # if lock.txt exits, hide select excel form
     # 如果 studta.xls文件存在，则显示之前的random连接
     # 如果 lock.txt文件存在，则不显示选取文件表单
-    show_previous=file_exists(os.path.join(app.config['UPLOAD_FOLDER'],app.config['STUDATA_FILE_NAME']))
-    lock_form = file_exists(os.path.join(app.config['UPLOAD_FOLDER'],app.config['LOCK_FILE_NAME']))
+    show_previous=file_exists(app.config.xls_file_path)
+    lock_form = file_exists(app.config.lock_file_path)
     return render_template('public/home.html', data_exist=show_previous, lock=lock_form)
 
 @blueprint.route('/about/')
@@ -41,34 +41,37 @@ def upload():
         app.logger.info(seed)
         app.logger.info(num)
         # check passwd
-        lock_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['LOCK_FILE_NAME'])
-        lock = file_exists(lock_file_path)
+        lock = file_exists(app.config.lock_file_path)
         if lock:
             passwd = request.form['passwd']
             app.logger.info(passwd)
             app.logger.info(lock)
-            passwd_file = open(lock_file_path,'r')
-            real_passwd = passwd_file.readline().strip()
-            passwd_file.close()
+            app.config.lock_file_lock.acquire()
+            passwd_file = open(app.config.lock_file_path,'r')
+            try:
+                real_passwd = passwd_file.readline().strip()
+            finally:
+                passwd_file.close()
+                app.config.lock_file_lock.release()
             if real_passwd != passwd:
                 return JSONR(ERRCODE.UNAUTHORIZED,'password was wrong')
-        else:
-            pass
-        seed_file_path = os.path.join(app.config['UPLOAD_FOLDER'], app.config['RANDOMSEED_FILE_NAME'])
-        num_file_path = os.path.join(app.config['UPLOAD_FOLDER'], app.config['RANDOMNUM_FILE_NAME'])
 
         # save random_seed
-        seed_file = open(seed_file_path,'w')
+        app.config.seed_file_lock.acquire()
+        seed_file = open(app.config.seed_file_path,'w')
         try:
             random_seed = int(seed)
             seed_file.write(str(random_seed))
         except ValueError:
             return JSONR(ERRCODE.FORMAT_ERROR,'random seed format error', seed)
         finally:
             seed_file.close()
+            app.config.seed_file_lock.release()
 
+        app.config.num_file_lock.acquire()
         # save random_num
-        num_file = open(num_file_path,'w')
+        num_file = open(app.config.num_file_path,'w')
+
         try:
             random_num = int(num)
             num_file.write(str(random_num))
@@ -77,67 +80,73 @@ def upload():
             return JSONR(ERRCODE.FORMAT_ERROR,'random num format error', num)
         finally:
             num_file.close()
+            app.config.num_file_lock.release()
 
         # save stu file
-        file = request.files['fileUploaded']
-        if file and allowed_file(file.filename,app.config['ALLOWED_EXTENSIONS']):
-            filename = secure_filename(file.filename)
-            # app.logger.info(filename.split('.',1)[0])
-            # target_filename = filename.split('.',1)[0] + '-' + str(hashlib.sha224(filename).hexdigest()) +'.'+ filename.split('.',1)[1]
-            # TODO only support xls type file
-            target_filename = app.config['STUDATA_FILE_NAME']
-            file.save(os.path.join(app.config['UPLOAD_FOLDER'], target_filename))
-            # return redirect(url_for('public.random',filename=filename))
-            return JSONR(ERRCODE.SUCCESS,'success')
-        else:
-            return JSONR(ERRCODE.INVALID_FILE,'invalid file extension')
+        app.config.xls_file_lock.acquire()
+        try:
+            file = request.files['fileUploaded']
+            if file and allowed_file(file.filename,app.config['ALLOWED_EXTENSIONS']):
+                filename = secure_filename(file.filename)
+                # app.logger.info(filename.split('.',1)[0])
+                # target_filename = filename.split('.',1)[0] + '-' + str(hashlib.sha224(filename).hexdigest()) +'.'+ filename.split('.',1)[1]
+                # TODO only support xls type file
+                target_filename = app.config['STUDATA_FILE_NAME']
+                file.save(os.path.join(app.config['UPLOAD_FOLDER'], target_filename))
+                # return redirect(url_for('public.random',filename=filename))
+                return JSONR(ERRCODE.SUCCESS,'success')
+            else:
+                return JSONR(ERRCODE.INVALID_FILE,'invalid file extension')
+        finally:
+            app.config.xls_file_lock.release()
+
     return JSONR(ERRCODE.INVALID_REQUEST,'only support post method')
 
 @blueprint.route('/random/', methods=['GET'])
 def random():
     """random page."""
     if file_exists(os.path.join(app.config['UPLOAD_FOLDER'],app.config['STUDATA_FILE_NAME'])):
-        seed_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMSEED_FILE_NAME'])
-        num_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMNUM_FILE_NAME'])
 
         random_num = 0
         random_seed =0
-        if file_exists(seed_file_path):
-            seed_file = open(seed_file_path)
+        app.config.seed_file_lock.acquire()
+        if file_exists(app.config.seed_file_path):
+            seed_file = open(app.config.seed_file_path)
             try:
                 seed = seed_file.readline().strip()
                 random_seed = int(seed)
             except ValueError:
                 return render_template('public/random.html',random_seed=0, random_num=0)
             finally:
                 seed_file.close()
+                app.config.seed_file_lock.release()
 
-        if file_exists(num_file_path):
-            num_file = open(num_file_path)
+        app.config.num_file_lock.acquire()
+        if file_exists(app.config.num_file_path):
+            num_file = open(app.config.num_file_path)
             try:
                 num = num_file.readline().strip()
                 random_num = int(num)
             except ValueError:
                 return render_template('public/random.html',random_seed=random_seed, random_num=0)
             finally:
                 num_file.close()
-        return render_template('public/random.html',random_seed=random_seed, random_num=random_num)
+                app.config.num_file_lock.release()
+            return render_template('public/random.html',random_seed=random_seed, random_num=random_num)
 
     return redirect(url_for('public.home'))
 
 
 @blueprint.route('/rand/', methods=['POST'])
 def rand():
     """rand"""
-    xls_file  = os.path.join(app.config['UPLOAD_FOLDER'],app.config['STUDATA_FILE_NAME'])
     # get random seed
-    seed_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMSEED_FILE_NAME'])
-    num_file_path = os.path.join(app.config['UPLOAD_FOLDER'],app.config['RANDOMNUM_FILE_NAME'])
 
     random_num = 0
     random_seed =0
-    if file_exists(seed_file_path):
-        seed_file = open(seed_file_path)
+    app.config.seed_file_lock.acquire()
+    if file_exists(app.config.seed_file_path):
+        seed_file = open(app.config.seed_file_path)
         seed = 0
         try:
             seed = seed_file.readline().strip()
@@ -146,8 +155,11 @@ def rand():
             return JSONR(ERRCODE.FORMAT_ERROR,'convert random seed failed',seed)
         finally:
             seed_file.close()
-    if file_exists(num_file_path):
-        num_file = open(num_file_path)
+            app.config.seed_file_lock.release()
+
+    app.config.num_file_lock.acquire()
+    if file_exists(app.config.num_file_path):
+        num_file = open(app.config.num_file_path)
         num = 0
         try:
             num = num_file.readline().strip()
@@ -156,17 +168,24 @@ def rand():
             return JSONR(ERRCODE.FORMAT_ERROR,'convert random num failed',num)
         finally:
             num_file.close()
+            app.config.num_file_lock.release()
+
     # calc random result
-    if file_exists(xls_file):
+    app.config.xls_file_lock.acquire()
+    if file_exists(app.config.xls_file_path):
        # rand
-       dicts = read_excel(xls_file)
-       res = RandomGenerator(random_seed, cal_range(dicts), random_num, dicts)
-       studs = res.GenerateResult()
-       res = []
-       for key in dicts:
-           res.append({"key":key,"value":dicts[key]})
-       app.logger.info(res)
-       data = {'allstus': res,'studs':studs }
-       return JSONR(ERRCODE.SUCCESS,'success',data)
+       try:
+           dicts = read_excel(app.config.xls_file_path)
+           res = RandomGenerator(random_seed, cal_range(dicts), random_num, dicts)
+           studs = res.GenerateResult()
+           res = []
+           for key in dicts:
+               res.append({"key":key,"value":dicts[key]})
+           app.logger.info(res)
+           data = {'allstus': res,'studs':studs }
+
+           return JSONR(ERRCODE.SUCCESS,'success',data)
+       finally:
+            app.config.xls_file_lock.release()
     return JSONR(ERRCODE.UNKNOW,'failed')
 

drls/settings.py

@@ -27,6 +27,8 @@ class Config(object):
 
 
 
+
+
 class ProdConfig(Config):
     """Production configuration."""
 

requirements/prod.txt

@@ -42,3 +42,6 @@ markdown2==2.3.5
 
 #Excel
 xlrd==1.1.0
+
+#filelock
+filelock==3.0.4

tests/test_rng.py

@@ -1,10 +1,12 @@
 import unittest
 
 from drls.rng import RandomGenerator
+from drls.utils import cal_range
 
 
 if __name__ == "__main__":
-    dict = {'21751117': 1.0,'21751114': 1.0,'21751115': 1.0, '21751110': 1.0,'21751111': 1.0, '21751464': 1.0, '21751465': 1.0, '21751462': 1.0, '21751463': 1.0}
-    res = RandomGenerator(1,9,5,dict)
+    dict = {'21751117': 100.0,'21751114': 1.0,'21751115': 1.0, '21751110': 1.0,'21751111': 1.0, '21751464': 1.0, '21751465': 1.0, '21751462': 1.0, '21751463': 1.0}
+    sum  = cal_range(dict)
+    res = RandomGenerator(1,sum,1,dict)
     studs = res.GenerateResult()
     print studs