Skip to content

Commit 0a57170

Browse files
wow-such-codewow-such-code
authored
prepare release (#112)
* add error handling via email for attachment upload (#111) * update readme for attachment upload
1 parent df5b42d commit 0a57170

File tree

2 files changed

+131
-66
lines changed

2 files changed

+131
-66
lines changed

‎README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,11 +212,15 @@ Incoming structure overview:
212212
Metadata is expected to be denoted in line-separated key-value pairs, where key and value are separated by a '='. The following structure/pairs are expected:
213213

214214
```
215-
user=<the (optional) uploading user name>
215+
user=<the (optional) uploading user name.>
216216
info=<short info about the file>
217217
barcode=<the sample code of the attachment sample>
218218
type=<the type of attachment: information or results>
219219
```
220+
If a university user name is provided and the registration of data fails, the user will receive an email containing the error.
221+
222+
The info field must not contain line breaks, as each line in the metadata file must contain a key-value pair.
223+
220224
The code of the attachment sample is built from the project code followed by three zeroes, conforming to the regular expression "Q[A-Z0-9]{4}000", e.g. QABCD000.
221225

222226
See code examples:

‎drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py‎

Lines changed: 126 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -22,81 +22,142 @@
2222
# *Q[Project Code]^4000*.*
2323
# *Q[Project Code]^4E[Experiment Number]-000*
2424
# IMPORTANT: ONLY PROJECT LEVEL WORKING RIGHT NOW
25-
ppattern = re.compile('Q\w{4}000')
25+
#ppattern = re.compile('Q\w{4}000')
2626
#epattern = re.compile('Q\w{4}E[1-9][0-9]*')
2727

28+
# contains properties used to send emails, e.g. the mail server and the "from" address to use
29+
import email_helper_qbic as email_helper
30+
# provides functionality used to send emails, e.g. about registration errors
31+
import etl_mailer
32+
33+
class MetadataFormattingException(Exception):
34+
"Thrown when metadata file cannot be successfully parsed."
35+
pass
36+
37+
class NoSamplesFoundForProjectException(Exception):
38+
"Thrown when sample cannot be found in openBIS."
39+
pass
40+
41+
class CouldNotCreateException(Exception):
42+
"Thrown when necessary experiment or sample could not be created."
43+
pass
44+
45+
def getInfoExperimentIdentifier(space, project):
46+
experimentID = '/' + space + '/' + project + '/'+ project+'_INFO'
47+
return experimentID
48+
49+
def extractProjectCode(sampleCode):
50+
return sampleCode[:5]
51+
2852
def process(transaction):
53+
error = None
54+
originalError = None
2955
context = transaction.getRegistrationContext().getPersistentMap()
3056

3157
# Get the incoming path of the transaction
3258
incomingPath = transaction.getIncoming().getAbsolutePath()
3359

34-
key = context.get("RETRY_COUNT")
35-
if (key == None):
36-
key = 1
60+
try:
61+
#read in the metadata file
62+
for f in os.listdir(incomingPath):
63+
if f == "metadata.txt":
64+
metadata = open(os.path.join(incomingPath, f))
65+
fileInfo = dict()
66+
for line in metadata:
67+
try:
68+
pair = line.strip().split('=')
69+
fileInfo[pair[0]] = pair[1]
70+
except IndexError as exception:
71+
originalError = exception
72+
error = MetadataFormattingException("Metadata file not correctly formatted. Check for additional line breaks.")
73+
metadata.close()
74+
try:
75+
user = fileInfo["user"]
76+
except:
77+
user = None
78+
secname = fileInfo["info"]
79+
sampleCode = fileInfo["barcode"]
80+
datasetType = fileInfo["type"]
81+
else:
82+
name = f
3783

38-
#read in the metadata file
39-
for f in os.listdir(incomingPath):
40-
if f == "metadata.txt":
41-
metadata = open(os.path.join(incomingPath, f))
42-
fileInfo = dict(line.strip().split('=') for line in metadata)
43-
metadata.close()
44-
try:
45-
user = fileInfo["user"]
46-
except:
47-
user = None
48-
secname = fileInfo["info"]
49-
code = fileInfo["barcode"]
50-
datasetType = fileInfo["type"]
51-
else:
52-
name = f
53-
54-
project = code[:5]
55-
type = "INFORMATION"
56-
if "Results" in datasetType:
57-
type = "RESULT"
58-
59-
if user:
60-
transaction.setUserId(user)
61-
62-
inputFile = os.path.join(incomingPath, name)
63-
newname = urllib.unquote(name)
64-
dataFile = os.path.join(incomingPath, newname)
65-
print "renaming "+inputFile+" to "+dataFile
66-
os.rename(inputFile, dataFile)
67-
68-
search_service = transaction.getSearchService()
69-
sc = SearchCriteria()
70-
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
71-
foundSamples = search_service.searchForSamples(sc)
72-
sample = None
73-
space = None
74-
sa = None
75-
attachmentReady = True
76-
77-
if len(foundSamples) == 0:
78-
attachmentReady = False
84+
project = extractProjectCode(sampleCode)
85+
type = "INFORMATION"
86+
if "Results" in datasetType:
87+
type = "RESULT"
88+
if error:
89+
raise error
90+
if user:
91+
transaction.setUserId(user)
92+
93+
inputFile = os.path.join(incomingPath, name)
94+
newname = urllib.unquote(name)
95+
dataFile = os.path.join(incomingPath, newname)
96+
print "renaming "+inputFile+" to "+dataFile
97+
os.rename(inputFile, dataFile)
98+
99+
search_service = transaction.getSearchService()
79100
sc = SearchCriteria()
80-
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, project+"ENTITY-1"))
101+
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
81102
foundSamples = search_service.searchForSamples(sc)
82-
sample = foundSamples[0]
83-
sampleID = sample.getSampleIdentifier()
84-
sa = transaction.getSampleForUpdate(sampleID)
85-
space = sa.getSpace()
86-
if not attachmentReady:
87-
infoSampleID = "/"+space+"/"+code
88-
sa = transaction.getSampleForUpdate(infoSampleID)
89-
if not sa:
90-
exp = transaction.createNewExperiment('/' + space + '/' + project + '/'+ project+'_INFO', "Q_PROJECT_DETAILS")
91-
sa = transaction.createNewSample('/' + space + '/'+ code, "Q_ATTACHMENT_SAMPLE")
92-
sa.setExperiment(exp)
93-
info = None
94-
95-
dataSet = transaction.createNewDataSet("Q_PROJECT_DATA")
96-
dataSet.setMeasuredData(False)
97-
dataSet.setPropertyValue("Q_SECONDARY_NAME", secname)
98-
dataSet.setPropertyValue("Q_ATTACHMENT_TYPE", type)
99-
dataSet.setSample(sa)
100-
transaction.moveFile(dataFile, dataSet)
103+
sample = None
104+
space = None
105+
sa = None
106+
attachmentSampleFound = True
101107

108+
if len(foundSamples) == 0:
109+
attachmentSampleFound = False
110+
sc = SearchCriteria()
111+
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, project+"ENTITY-1"))
112+
foundSamples = search_service.searchForSamples(sc)
113+
try:
114+
sample = foundSamples[0]
115+
except IndexError as exception:
116+
originalError = exception
117+
error = NoSamplesFoundForProjectException("No sample could be found for this project.")
118+
sampleID = sample.getSampleIdentifier()
119+
sa = transaction.getSampleForUpdate(sampleID)
120+
space = sa.getSpace()
121+
if not attachmentSampleFound:
122+
# fetch it by name
123+
infoSampleID = "/"+space+"/"+sampleCode
124+
sa = transaction.getSampleForUpdate(infoSampleID)
125+
if not sa:
126+
# create necessary objects if sample really doesn't exist
127+
try:
128+
experimentID = getInfoExperimentIdentifier(space, project)
129+
exp = transaction.createNewExperiment(experimentID, "Q_PROJECT_DETAILS")
130+
except Exception as exception:
131+
originalError = exception
132+
error = CouldNotCreateException("Experiment "+experimentID+" could not be created.")
133+
try:
134+
sa = transaction.createNewSample(infoSampleID, "Q_ATTACHMENT_SAMPLE")
135+
except Exception as exception:
136+
originalError = exception
137+
error = CouldNotCreateException("Sample "+infoSampleID+" was not found and could not be created.")
138+
sa.setExperiment(exp)
102139

140+
dataSet = transaction.createNewDataSet("Q_PROJECT_DATA")
141+
dataSet.setMeasuredData(False)
142+
dataSet.setPropertyValue("Q_SECONDARY_NAME", secname)
143+
dataSet.setPropertyValue("Q_ATTACHMENT_TYPE", type)
144+
dataSet.setSample(sa)
145+
transaction.moveFile(dataFile, dataSet)
146+
# catch other, unknown exceptions
147+
except Exception as exception:
148+
originalError = exception
149+
if not error:
150+
error = Exception("Unknown exception occured: "+str(exception))
151+
if originalError:
152+
# if there is a problem sending the email, we log this and raise the original exception
153+
try:
154+
mailFactory = etl_mailer.EmailFactory()
155+
etlMailer = etl_mailer.ETLMailer(email_helper.get_mail_host(), email_helper.get_mail_server(), email_helper.get_mail_from())
156+
157+
subject = "Error storing project attachment uploaded for "+project
158+
content = mailFactory.formatRegistrationErrorContent(str(error))
159+
etlMailer.send_email([user], subject, content)
160+
except Exception as mailException:
161+
print "Could not send error email: "+str(mailException)
162+
pass
163+
raise originalError

0 commit comments

Comments
 (0)