Storage pdf to obs (#88)

* add obs as the storage for pdf

* change the config file to yaml

* use obs as the pdf storage

* implement list corps which have uploaded pdf

* minor update

Dockerfile

@@ -13,6 +13,7 @@ COPY ./conf /opt/app/conf
 COPY ./util/merge-signature.py /opt/app/util/merge-signature.py
 # overwrite config yaml
 COPY ./deploy/app.conf /opt/app/conf
+COPY ./deploy/app.conf.yaml /opt/app/conf
 COPY  --from=BUILDER /go/src/github.com/opensourceways/app-cla-server/cla-server /opt/app
 
 WORKDIR /opt/app/

conf/app.conf

@@ -5,30 +5,4 @@ autorender = false
 copyrequestbody = true
 EnableDocs = true
 
-python_bin = /usr/bin/python3
-
-cla_fields_number = 10
-cla_platform_url = https://clasign.osinfra.cn
-
-verification_code_expiry = 300
-api_token_expiry = 3600
-api_token_key = fsfsfsafsfsasaf242342424sdfs;.]{77&&&
-symmetric_encryption_key = key-can-be--16-24-32-bytes-long!
-
-pdf_org_signature_dir = ./conf/org_signature_pdf
-pdf_out_dir = ./conf/pdf
-
-code_platforms = ./conf/code_platforms.yaml
-email_platforms = ./conf/email.yaml
-
-employee_managers_number = 5
-
-[mongodb]
-mongodb_conn = mongodb://root:123@localhost:32768
-mongodb_db = cla
-link_collection = links
-corp_pdf_collection = corp_pdfs
-org_email_collection = org_emails
-verification_code_collection = verification_codes
-corp_signing_collection = corp_signings
-individual_signing_collection = individual_signings
+app_conf = ./conf/app.conf.yaml

conf/app.conf.yaml

@@ -0,0 +1,32 @@
+python_bin: /usr/bin/python3
+
+cla_fields_number: 10
+cla_platform_url: https://clasign.osinfra.cn
+
+verification_code_expiry: 300
+api_token_expiry: 3600
+api_token_key: fsfsfsafsfsasaf242342424sdfs;.]{77&&&
+symmetric_encryption_key: key-can-be--16-24-32-bytes-long!
+
+pdf_org_signature_dir: ./conf/org_signature_pdf
+pdf_out_dir: ./conf/pdf
+
+code_platforms: ./conf/code_platforms.yaml
+email_platforms: ./conf/email.yaml
+
+employee_managers_number: 5
+
+mongodb:
+  mongodb_conn: mongodb://root:123@localhost:32768
+  mongodb_db: cla
+  link_collection: links
+  corp_pdf_collection: corp_pdfs
+  org_email_collection: org_emails
+  verification_code_collection: verification_codes
+  corp_signing_collection: corp_signings
+  individual_signing_collection: individual_signings
+
+obs:
+  name: huaweicloud-obs
+  bucket: cla
+  credential_file: ./conf/huaweicloud_obs_credential.yaml

conf/huaweicloud_obs_credential.yaml

@@ -0,0 +1,4 @@
+access_key: {{ access key }}
+secret_key: {{ secret key }}
+endpoint: https://obs.{{ region }}.myhuaweicloud.com
+object_encryption_key: "{{ AES256 encryption key }}"

config/config.go

@@ -3,13 +3,21 @@ package config
 import (
 	"fmt"
 
-	"github.com/astaxie/beego"
-	"github.com/huaweicloud/golangsdk"
-
 	"github.com/opensourceways/app-cla-server/util"
 )
 
-var AppConfig *appConfig
+var AppConfig = &appConfig{}
+
+func InitAppConfig(path string) error {
+	cfg := AppConfig
+	if err := util.LoadFromYaml(path, cfg); err != nil {
+		return err
+	}
+
+	cfg.setDefault()
+
+	return cfg.validate()
+}
 
 type appConfig struct {
 	PythonBin                string        `json:"python_bin" required:"true"`
@@ -29,6 +37,7 @@ type appConfig struct {
 	EmployeeManagersNumber   int           `json:"employee_managers_number" required:"true"`
 	CLAPlatformURL           string        `json:"cla_platform_url" required:"true"`
 	Mongodb                  MongodbConfig `json:"mongodb" required:"true"`
+	OBS                      OBS           `json:"obs" required:"true"`
 }
 
 type MongodbConfig struct {
@@ -42,112 +51,72 @@ type MongodbConfig struct {
 	IndividualSigningCollection string `json:"individual_signing_collection" required:"true"`
 }
 
-func InitAppConfig() error {
-	claFieldsNumber, err := beego.AppConfig.Int("cla_fields_number")
-	if err != nil {
-		return err
-	}
-
-	maxSizeOfCorpCLAPDF := beego.AppConfig.DefaultInt("max_size_of_corp_cla_pdf", (2 << 20))
-	maxSizeOfOrgSignaturePDF := beego.AppConfig.DefaultInt("max_size_of_org_signature_pdf", (1 << 20))
-	minLengthOfPassword := beego.AppConfig.DefaultInt("min_length_of_password", 6)
-	maxLengthOfPassword := beego.AppConfig.DefaultInt("max_length_of_password", 16)
+type OBS struct {
+	Name           string `json:"name" required:"true"`
+	Bucket         string `json:"bucket" required:"true"`
+	CredentialFile string `json:"credential_file" required:"true"`
+}
 
-	tokenExpiry, err := beego.AppConfig.Int64("api_token_expiry")
-	if err != nil {
-		return err
+func (cfg *appConfig) setDefault() {
+	if cfg.MaxSizeOfCorpCLAPDF <= 0 {
+		cfg.MaxSizeOfCorpCLAPDF = (5 << 20)
 	}
-
-	codeExpiry, err := beego.AppConfig.Int64("verification_code_expiry")
-	if err != nil {
-		return err
+	if cfg.MaxSizeOfOrgSignaturePDF <= 0 {
+		cfg.MaxSizeOfOrgSignaturePDF = (1 << 20)
 	}
 
-	employeeMangers, err := beego.AppConfig.Int("employee_managers_number")
-	if err != nil {
-		return err
+	if cfg.MinLengthOfPassword <= 0 {
+		cfg.MinLengthOfPassword = 6
 	}
 
-	AppConfig = &appConfig{
-		PythonBin:                beego.AppConfig.String("python_bin"),
-		CLAFieldsNumber:          claFieldsNumber,
-		MaxSizeOfCorpCLAPDF:      maxSizeOfCorpCLAPDF,
-		MaxSizeOfOrgSignaturePDF: maxSizeOfOrgSignaturePDF,
-		MinLengthOfPassword:      minLengthOfPassword,
-		MaxLengthOfPassword:      maxLengthOfPassword,
-		VerificationCodeExpiry:   codeExpiry,
-		APITokenExpiry:           tokenExpiry,
-		APITokenKey:              beego.AppConfig.String("api_token_key"),
-		SymmetricEncryptionKey:   beego.AppConfig.String("symmetric_encryption_key"),
-		PDFOrgSignatureDir:       beego.AppConfig.String("pdf_org_signature_dir"),
-		PDFOutDir:                beego.AppConfig.String("pdf_out_dir"),
-		CodePlatformConfigFile:   beego.AppConfig.String("code_platforms"),
-		EmailPlatformConfigFile:  beego.AppConfig.String("email_platforms"),
-		EmployeeManagersNumber:   employeeMangers,
-		CLAPlatformURL:           beego.AppConfig.String("cla_platform_url"),
-		Mongodb: MongodbConfig{
-			MongodbConn:                 beego.AppConfig.String("mongodb::mongodb_conn"),
-			DBName:                      beego.AppConfig.String("mongodb::mongodb_db"),
-			LinkCollection:              beego.AppConfig.String("mongodb::link_collection"),
-			OrgEmailCollection:          beego.AppConfig.String("mongodb::org_email_collection"),
-			CorpPDFCollection:           beego.AppConfig.String("mongodb::corp_pdf_collection"),
-			VCCollection:                beego.AppConfig.String("mongodb::verification_code_collection"),
-			CorpSigningCollection:       beego.AppConfig.String("mongodb::corp_signing_collection"),
-			IndividualSigningCollection: beego.AppConfig.String("mongodb::individual_signing_collection"),
-		},
-	}
-	return AppConfig.validate()
-}
-
-func (this *appConfig) validate() error {
-	_, err := golangsdk.BuildRequestBody(this, "")
-	if err != nil {
-		return fmt.Errorf("config file error: %s", err.Error())
+	if cfg.MaxLengthOfPassword <= 0 {
+		cfg.MaxLengthOfPassword = 16
 	}
+}
 
-	if util.IsFileNotExist(this.PythonBin) {
-		return fmt.Errorf("The file:%s is not exist", this.PythonBin)
+func (cfg *appConfig) validate() error {
+	if util.IsFileNotExist(cfg.PythonBin) {
+		return fmt.Errorf("The file:%s is not exist", cfg.PythonBin)
 	}
 
-	if this.CLAFieldsNumber <= 0 {
-		return fmt.Errorf("The cla_fields_number:%d should be bigger than 0", this.CLAFieldsNumber)
+	if cfg.CLAFieldsNumber <= 0 {
+		return fmt.Errorf("The cla_fields_number:%d should be bigger than 0", cfg.CLAFieldsNumber)
 	}
 
-	if this.VerificationCodeExpiry <= 0 {
-		return fmt.Errorf("The verification_code_expiry:%d should be bigger than 0", this.VerificationCodeExpiry)
+	if cfg.VerificationCodeExpiry <= 0 {
+		return fmt.Errorf("The verification_code_expiry:%d should be bigger than 0", cfg.VerificationCodeExpiry)
 	}
 
-	if this.APITokenExpiry <= 0 {
-		return fmt.Errorf("The apit_oken_expiry:%d should be bigger than 0", this.APITokenExpiry)
+	if cfg.APITokenExpiry <= 0 {
+		return fmt.Errorf("The apit_oken_expiry:%d should be bigger than 0", cfg.APITokenExpiry)
 	}
 
-	if this.EmployeeManagersNumber <= 0 {
-		return fmt.Errorf("The employee_managers_number:%d should be bigger than 0", this.EmployeeManagersNumber)
+	if cfg.EmployeeManagersNumber <= 0 {
+		return fmt.Errorf("The employee_managers_number:%d should be bigger than 0", cfg.EmployeeManagersNumber)
 	}
 
-	if len(this.APITokenKey) < 20 {
+	if len(cfg.APITokenKey) < 20 {
 		return fmt.Errorf("The length of api_token_key should be bigger than 20")
 	}
 
-	if err := util.IsSymmetricEncryptionKeyValid(this.SymmetricEncryptionKey); err != nil {
+	if err := util.IsSymmetricEncryptionKeyValid(cfg.SymmetricEncryptionKey); err != nil {
 		return fmt.Errorf("The symmetric encryption key is not valid, %s", err.Error())
 	}
 
-	if util.IsNotDir(this.PDFOrgSignatureDir) {
-		return fmt.Errorf("The directory:%s is not exist", this.PDFOrgSignatureDir)
+	if util.IsNotDir(cfg.PDFOrgSignatureDir) {
+		return fmt.Errorf("The directory:%s is not exist", cfg.PDFOrgSignatureDir)
 	}
 
-	if util.IsNotDir(this.PDFOutDir) {
-		return fmt.Errorf("The directory:%s is not exist", this.PDFOutDir)
-
+	if util.IsNotDir(cfg.PDFOutDir) {
+		return fmt.Errorf("The directory:%s is not exist", cfg.PDFOutDir)
 	}
 
-	if util.IsFileNotExist(this.CodePlatformConfigFile) {
-		return fmt.Errorf("The file:%s is not exist", this.CodePlatformConfigFile)
+	if util.IsFileNotExist(cfg.CodePlatformConfigFile) {
+		return fmt.Errorf("The file:%s is not exist", cfg.CodePlatformConfigFile)
 	}
 
-	if util.IsFileNotExist(this.EmailPlatformConfigFile) {
-		return fmt.Errorf("The file:%s is not exist", this.EmailPlatformConfigFile)
+	if util.IsFileNotExist(cfg.EmailPlatformConfigFile) {
+		return fmt.Errorf("The file:%s is not exist", cfg.EmailPlatformConfigFile)
 	}
 
 	return nil

controllers/corporation-pdf.go

@@ -35,25 +35,22 @@ func (this *CorporationPDFController) downloadCorpPDF(linkID, corpEmail string)
 	if err != nil {
 		return newFailedApiResult(500, errSystemError, err)
 	}
+	path := f.Name()
+	f.Close()
+
 	defer func() {
-		f.Close()
-		os.Remove(f.Name())
+		os.Remove(path)
 	}()
 
-	pdf, merr := models.DownloadCorporationSigningPDF(linkID, corpEmail)
+	merr := models.DownloadCorporationSigningPDF(linkID, corpEmail, path)
 	if merr != nil {
 		if merr.IsErrorOf(models.ErrNoLinkOrUnuploaed) {
 			return newFailedApiResult(400, errUnuploaded, merr)
 		}
 		return parseModelError(merr)
 	}
 
-	if _, err = f.Write(*pdf); err != nil {
-		return newFailedApiResult(500, errSystemError, err)
-	}
-
-	f.Close()
-	this.downloadFile(f.Name())
+	this.downloadFile(path)
 	return nil
 }
 
@@ -102,7 +99,7 @@ func (this *CorporationPDFController) Upload() {
 		return
 	}
 
-	if err := models.UploadCorporationSigningPDF(linkID, corpEmail, &data); err != nil {
+	if err := models.UploadCorporationSigningPDF(linkID, corpEmail, data); err != nil {
 		this.sendModelErrorAsResp(err, action)
 		return
 	}

dbmodels/db.go

@@ -11,6 +11,11 @@ func GetDB() IDB {
 }
 
 type IDB interface {
+	IModel
+	IFile
+}
+
+type IModel interface {
 	ILink
 	ICorporationSigning
 	ICorporationManager
@@ -29,11 +34,13 @@ type ICorporationSigning interface {
 	ListDeletedCorpSignings(linkID string) ([]CorporationSigningBasicInfo, IDBError)
 	GetCorpSigningDetail(linkID, email string) ([]Field, *CorpSigningCreateOpt, IDBError)
 	GetCorpSigningBasicInfo(linkID, email string) (*CorporationSigningBasicInfo, IDBError)
+}
 
-	UploadCorporationSigningPDF(linkID string, adminEmail string, pdf *[]byte) IDBError
-	DownloadCorporationSigningPDF(linkID string, email string) (*[]byte, IDBError)
-	IsCorpSigningPDFUploaded(linkID string, email string) (bool, IDBError)
-	ListCorpsWithPDFUploaded(linkID string) ([]string, IDBError)
+type IFile interface {
+	UploadCorporationSigningPDF(linkID, adminEmail string, pdf []byte) IDBError
+	DownloadCorporationSigningPDF(linkID, email, path string) IDBError
+	IsCorporationSigningPDFUploaded(linkID, email string) (bool, IDBError)
+	ListCorporationsWithPDFUploaded(linkID string) ([]string, IDBError)
 }
 
 type ICorporationManager interface {

dbmodels/error.go

@@ -14,3 +14,27 @@ const (
 	ErrRecordExists      DBErrCode = "db_record_exists"
 	ErrMarshalDataFaield DBErrCode = "failed_to_marshal_data"
 )
+
+type dbError struct {
+	code DBErrCode
+	err  error
+}
+
+func (e dbError) Error() string {
+	if e.err == nil {
+		return ""
+	}
+	return e.err.Error()
+}
+
+func (e dbError) IsErrorOf(code DBErrCode) bool {
+	return e.code == code
+}
+
+func (e dbError) ErrCode() DBErrCode {
+	return e.code
+}
+
+func NewDBError(code DBErrCode, err error) IDBError {
+	return dbError{code: code, err: err}
+}

deploy/app.conf

@@ -5,35 +5,4 @@ autorender = false
 copyrequestbody = true
 EnableDocs = false
 
-python_bin = /usr/bin/python3
-
-cla_fields_number = 10
-
-verification_code_expiry = 300
-api_token_expiry = 1800
-api_token_key = "${API_TOKEN_KEY}"
-symmetric_encryption_key = "${SYMMETRIC_ENCRYPTION_KEY}"
-
-pdf_org_signature_dir = ./conf/pdfs/org_signature_pdf
-pdf_out_dir = ./conf/pdfs/output
-
-code_platforms = ./conf/platforms/code_platforms.yaml
-email_platforms = ./conf/platforms/email.yaml
-
-employee_managers_number = 5
-
-max_size_of_corp_cla_pdf = 5242880
-max_size_of_org_signature_pdfa = 204800
-
-cla_platform_url = "${CLA_PLATFORM_URL}"
-
-[mongodb]
-mongodb_conn = "${MONGODB_CONNECTION}"
-mongodb_db = "${MONGODB_NAME}"
-
-link_collection = links
-corp_pdf_collection = corp_pdfs
-org_email_collection = org_emails
-verification_code_collection = verification_codes
-corp_signing_collection = corp_signings
-individual_signing_collection = individual_signings
+app_conf = ./conf/app.conf.yaml