F1

cell_class/CellClassification.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#VERSION v0.1
+
+#Find NDPI etc and extract them
+
+#$1 Source directory
+#$2 Target
+
+# bash CellClassification.sh `pwd`
+# find . -name "*.bsubclassification" -exec bash -c "bsub < {}" \;
+
+SAMPLES=`find ${1} -type d -name "*.ndpi"`
+FULLDIRNAME=${1}
+mkdir -p "${1}/bsubclassification"
+mkdir -p "${1}/errorsclassification"
+mkdir -p "${1}/outputsclassification"
+
+cp /mnt/scratch/users/molecpath/sraza/scripts/MyCodes/TracerX/classification/20171019-SCCNNClassifier/parameters-classification.txt "${1}"
+FULLDIRNAMEESC=$(echo $FULLDIRNAME | sed 's_/_\\/_g') 
+for s in $SAMPLES; do	
+  echo "Create bsub for ${s}"
+  BNAME=`basename ${s}`
+  cp /mnt/scratch/users/molecpath/sraza/scripts/MyCodes/TracerX/classification/20171019-SCCNNClassifier/header_cell_classification "${FULLDIRNAME}/bsubclassification/${BNAME}_extract.bsubclassification"  
+  sed -i "s/###NAME###/${BNAME}/g" "${FULLDIRNAME}/bsubclassification/${BNAME}_extract.bsubclassification"
+  sed -i "s/###DIRNAME###/${FULLDIRNAMEESC}/g" "${FULLDIRNAME}/bsubclassification/${BNAME}_extract.bsubclassification"
+done	
+

cell_class/Generate_Output.py

@@ -0,0 +1,48 @@
+import os
+
+import sccnn_classifier as sccnn_classifier
+from subpackages import NetworkOptions
+
+# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+
+opts = NetworkOptions.NetworkOptions(exp_dir='ExpDir-IHC/',
+                                     num_examples_per_epoch_train=1,
+                                     num_examples_per_epoch_valid=1,
+                                     image_height=51,
+                                     image_width=51,
+                                     in_feat_dim=3,
+                                     in_label_dim=1,
+                                     num_of_classes=4,
+                                     batch_size=500,
+                                     data_dir='R:\\tracerx\\Melanoma\\Quad\\data\\cws',
+                                     results_dir='D:/tmp/results_diagostics-ihc/classification',
+                                     detection_results_path='R:\\tracerx\\Melanoma\\Quad\\results\\detection\\20171017',
+                                     tissue_segment_dir='',
+                                     preprocessed_dir=None,
+                                     current_epoch_num=0,
+                                     file_name_pattern='CB12*',
+                                     pre_process=True,
+                                     color_code_file='IHC_CD4_CD8_FoxP3.csv')
+
+opts.results_dir = (os.path.join(opts.results_dir, '20171019'))
+opts.preprocessed_dir = os.path.join(opts.preprocessed_dir, '20171019')
+
+if not os.path.isdir(opts.results_dir):
+    os.makedirs(opts.results_dir)
+if not os.path.isdir(os.path.join(opts.results_dir, 'mat')):
+    os.makedirs(os.path.join(opts.results_dir, 'mat'))
+if not os.path.isdir(os.path.join(opts.results_dir, 'annotated_images')):
+    os.makedirs(os.path.join(opts.results_dir, 'annotated_images'))
+if not os.path.isdir(os.path.join(opts.results_dir, 'csv')):
+    os.makedirs(os.path.join(opts.results_dir, 'csv'))
+if not os.path.isdir(os.path.join(opts.preprocessed_dir, 'pre_processed')):
+    os.makedirs(os.path.join(opts.preprocessed_dir, 'pre_processed'))
+
+Network = sccnn_classifier.SccnnClassifier(batch_size=opts.batch_size,
+                                           image_height=opts.image_height,
+                                           image_width=opts.image_width,
+                                           in_feat_dim=opts.in_feat_dim,
+                                           in_label_dim=opts.in_label_dim,
+                                           num_of_classes=opts.num_of_classes)
+Network.generate_output(opts=opts)

cell_class/Generate_Output_Davros.py

@@ -0,0 +1,77 @@
+import os
+import sys
+import sccnn_classifier as sccnn_classifier
+from subpackages import NetworkOptions
+
+
+data_dir = sys.argv[1]
+sub_dir_name = sys.argv[2]
+
+d = {'tissue_segment_dir': '',
+     'preprocessed_dir': None,
+     'exp_dir': 'ExpDir',
+     'num_of_classes': 4,
+     'color_code_file': 'color_code_file.csv'}
+with open(os.path.join(data_dir, 'parameters-classification.txt')) as param:
+    for line in param:
+        a = line.split(' ')
+        d[a[0]] = a[1].strip('\n')
+
+print('results_dir: ' + d['results_dir'], flush=True)
+print('tissue_segment_dir: ' + d['tissue_segment_dir'], flush=True)
+print('detection_results_path:' + d['detection_results_path'], flush=True)
+print('file_name_pattern: ' + d['file_name_pattern'], flush=True)
+print('date: ' + d['date'], flush=True)
+print('exp_dir: ' + d['exp_dir'], flush=True)
+
+opts = NetworkOptions.NetworkOptions(exp_dir=d['exp_dir'],
+                                     num_examples_per_epoch_train=1,
+                                     num_examples_per_epoch_valid=1,
+                                     image_height=51,
+                                     image_width=51,
+                                     in_feat_dim=int(d['in_feat_dim']),
+                                     in_label_dim=1,
+                                     num_of_classes=int(d['num_of_classes']),
+                                     batch_size=500,
+                                     data_dir=data_dir,
+                                     results_dir=d['results_dir'],
+                                     detection_results_path=d['detection_results_path'],
+                                     tissue_segment_dir=d['tissue_segment_dir'],
+                                     preprocessed_dir=d['preprocessed_dir'],
+                                     current_epoch_num=0,
+                                     file_name_pattern=d['file_name_pattern'],
+                                     color_code_file=d['color_code_file'],
+                                     pre_process=True)
+
+opts.results_dir = os.path.join(opts.results_dir, d['date'])
+opts.preprocessed_dir = os.path.join(opts.preprocessed_dir, d['date'])
+
+if not os.path.isdir(opts.results_dir):
+    os.makedirs(opts.results_dir, exist_ok=True)
+if not os.path.isdir(os.path.join(opts.results_dir, 'mat')):
+    os.makedirs(os.path.join(opts.results_dir, 'mat'), exist_ok=True)
+if not os.path.isdir(os.path.join(opts.results_dir, 'annotated_images')):
+    os.makedirs(os.path.join(opts.results_dir, 'annotated_images'), exist_ok=True)
+if not os.path.isdir(os.path.join(opts.results_dir, 'csv')):
+    os.makedirs(os.path.join(opts.results_dir, 'csv'), exist_ok=True)
+if not os.path.isdir(os.path.join(opts.preprocessed_dir, 'pre_processed')):
+    os.makedirs(os.path.join(opts.preprocessed_dir, 'pre_processed'), exist_ok=True)
+
+Network = sccnn_classifier.SccnnClassifier(batch_size=opts.batch_size,
+                                           image_height=opts.image_height,
+                                           image_width=opts.image_width,
+                                           in_feat_dim=opts.in_feat_dim,
+                                           in_label_dim=opts.in_label_dim,
+                                           num_of_classes=opts.num_of_classes)
+
+print('\n\n\n', flush=True)
+print('opts.data_dir:' + os.path.join(opts.data_dir, sub_dir_name), flush=True)
+print('opts.results_dir:' + os.path.join(opts.results_dir, sub_dir_name), flush=True)
+print('opts.detection_results_path:' + os.path.join(opts.detection_results_path, sub_dir_name), flush=True)
+print('opts.preprocessed_dir:' + os.path.join(opts.preprocessed_dir, 'pre_processed', sub_dir_name), flush=True)
+print('opts.tissue_segmentation:' + os.path.join(opts.tissue_segment_dir, sub_dir_name), flush=True)
+print('opts.file_name_pattern:' + opts.file_name_pattern, flush=True)
+print('opts.pre_process:' + str(opts.pre_process), flush=True)
+print('opts.exp_dir:' + opts.exp_dir, flush=True)
+
+Network.generate_output_sub_dir(opts=opts, sub_dir_name=sub_dir_name, network_output=True)

cell_class/SaveClassificationOutput.m

@@ -0,0 +1,44 @@
+clc
+clear
+addpath('C:\Users\adminsraza\Documents\MATLAB\export_fig');
+addpath matlab\
+%%
+filepath = 'D:\Shan\MyCodes\TracerX\CellDetection\SCCNN\SCCNN_v2\ExpDir\TCGA-05-4389-01A-01-BS1';
+classifier_output = 'D:\Shan\MyCodes\TracerX\CellDetection\SCCNN\SCCNN_v2\ExpDir\TCGA-05-4389-01A-01-BS1';
+imagefolder = 'Z:\TCGA\Lung\data\raw\LUAD_20x\TCGA-05-4389-01A-01-BS1';
+files = dir(fullfile(filepath, '*.csv'));
+%%
+warning('off');
+if ~exist(fullfile(classifier_output, 'classification'), 'dir')
+    mkdir(fullfile(classifier_output, 'classification'));
+end
+parfor i = 1:length(files)
+    fprintf('%s\n', files(i).name);
+    A = importdata(fullfile(files(i).folder, files(i).name));
+    image = imread(fullfile(imagefolder, [files(i).name(1:end-3), 'jpg']));
+    h = figure('Visible', 'off');
+    warning('off', 'Images:initSize:adjustingMag');
+    imshow(image,[]);
+    if isfield(A, 'data')
+        detection = A.data;
+        mat = load(fullfile(classifier_output, [files(i).name(1:end-4), ...
+            '_classification.mat']));
+        cell_ids = mat.cell_ids;
+        output = mat.output;
+        C = unique(cell_ids);
+        class = zeros(length(C),1);
+        for j = 1:length(C)
+            class(j) = mode(mat.output(mat.cell_ids==C(j)));
+        end
+        hold on;
+        plot(detection(class==1,1),detection(class==1,2),'.y','markersize',5);
+        plot(detection(class==2,1),detection(class==2,2),'.b','markersize',5);
+        plot(detection(class==3,1),detection(class==3,2),'.g','markersize',5);
+        plot(detection(class==4,1),detection(class==4,2),'.r','markersize',5);
+        hold off;
+    end
+    imagedata = export_fig(gca, '-m3');
+    imagedata = imresize(imagedata, [size(image,1),size(image,2)]);
+    imwrite(imagedata, fullfile(classifier_output, 'annotated_images', [mat_file_name(1:end-3), 'png']), 'png');
+    close(gcf);
+end

cell_class/__init__.py

@@ -0,0 +1 @@
+import subpackages

cell_class/confusionmatStats.m

@@ -0,0 +1,70 @@
+function stats = confusionmatStats(group,grouphat)
+% INPUT
+% group = true class labels
+% grouphat = predicted class labels
+%
+% OR INPUT
+% stats = confusionmatStats(group);
+% group = confusion matrix from matlab function (confusionmat)
+%
+% OUTPUT
+% stats is a structure array
+% stats.confusionMat
+%               Predicted Classes
+%                    p'    n'
+%              ___|_____|_____| 
+%       Actual  p |     |     |
+%      Classes  n |     |     |
+%
+% stats.accuracy = (TP + TN)/(TP + FP + FN + TN) ; the average accuracy is returned
+% stats.precision = TP / (TP + FP)                  % for each class label
+% stats.sensitivity = TP / (TP + FN)                % for each class label
+% stats.specificity = TN / (FP + TN)                % for each class label
+% stats.recall = sensitivity                        % for each class label
+% stats.Fscore = 2*TP /(2*TP + FP + FN)            % for each class label
+%
+% TP: true positive, TN: true negative, 
+% FP: false positive, FN: false negative
+% 
+
+field1 = 'confusionMat';
+if nargin < 2
+    value1 = group;
+else
+    [value1,gorder] = confusionmat(group,grouphat);
+end
+
+numOfClasses = size(value1,1);
+totalSamples = sum(sum(value1));
+
+[TP,TN,FP,FN,accuracy,sensitivity,specificity,precision,f_score] = deal(zeros(numOfClasses,1));
+for class = 1:numOfClasses
+   TP(class) = value1(class,class);
+   tempMat = value1;
+   tempMat(:,class) = []; % remove column
+   tempMat(class,:) = []; % remove row
+   TN(class) = sum(sum(tempMat));
+   FP(class) = sum(value1(:,class))-TP(class);
+   FN(class) = sum(value1(class,:))-TP(class);
+end
+
+for class = 1:numOfClasses
+    accuracy(class) = (TP(class) + TN(class)) / totalSamples;
+    sensitivity(class) = TP(class) / (TP(class) + FN(class));
+    specificity(class) = TN(class) / (FP(class) + TN(class));
+    precision(class) = TP(class) / (TP(class) + FP(class));
+    f_score(class) = 2*TP(class)/(2*TP(class) + FP(class) + FN(class));
+end
+
+field2 = 'accuracy';  value2 = accuracy;
+field3 = 'sensitivity';  value3 = sensitivity;
+field4 = 'specificity';  value4 = specificity;
+field5 = 'precision';  value5 = precision;
+field6 = 'recall';  value6 = sensitivity;
+field7 = 'Fscore';  value7 = f_score;
+field8 = 'ConfusionMatNormalized'; value8 = value1./sum(value1,2);
+stats = struct(field1,value1,field2,value2,field3,value3,field4,value4,field5,value5,field6,value6,field7,value7, field8, value8);
+if exist('gorder','var')
+    stats = struct(field1,value1,field2,value2,field3,value3,field4,value4,field5,value5,field6,value6,field7,value7, field8, value8,'groupOrder',gorder);
+end
+

cell_class/header_cell_classification

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+#BSUB -J "###NAME###"
+#BSUB -P gpu-service
+#BSUB -q yuangpu
+#BSUB -e errorsclassification/###NAME###.errors.%J
+#BSUB -o outputsclassification/###NAME###.output.%J
+#BSUB -R "span[hosts=1]"
+#BSUB -R "rusage[ngpus_excl_p=1]"
+#BSUB -W 50:00
+#BSUB -u [email protected]
+module load anaconda/3/4.3.0
+module load gcc/4.9.3
+source activate tfDavrosGPU1p4
+cd /mnt/scratch/users/molecpath/sraza/scripts/MyCodes/TracerX/classification/20171019-SCCNNClassifier
+python Generate_Output_Davros.py "###DIRNAME###" "###NAME###"

cell_class/header_cell_classification_davros

@@ -0,0 +1,15 @@
+#BSUB -J "###NAME###"
+#BSUB -P DMPYXYAAO
+#BSUB -e errorsclassification/###NAME###.errors.%J
+#BSUB -o outputsclassification/###NAME###.output.%J
+#BSUB -n 1
+#BSUB -N
+#BSUB -W 50:00
+#BSUB -R "span[hosts=1]"
+#BSUB -q normal
+#BSUB -u [email protected]
+module load anaconda/3/4.3.0
+module load gcc/4.9.3
+source activate tfdavrosCPU1p3
+cd /mnt/scratch/users/molecpath/sraza/scripts/MyCodes/TracerX/classification/20171019-SCCNNClassifier
+python Generate_Output_Davros.py "###DIRNAME###" "###NAME###"

cell_class/initialize_matlab_variables.m

@@ -0,0 +1,8 @@
+clear
+%%
+addpath(genpath('matlab/'));
+matlab_input = {};
+mat_file_name = '';
+results_dir = '';
+image_path = '';
+csv_detection_results_dir = '';

cell_class/matlab/Save_Classification_Output.m

@@ -0,0 +1,36 @@
+function Save_Classification_Output(results_dir, sub_dir_name, mat_file_name, image_path_full, csv_file_name, color_code_file)
+if ~exist(fullfile(results_dir, 'annotated_images', sub_dir_name, [mat_file_name(1:end-3), 'png']), 'file')
+    strength = 5;
+    A = readtable(csv_file_name);
+    image = imread(image_path_full);
+    class = [];
+    colorcodes = readtable(fullfile(fileparts(mfilename('fullpath')), 'colorcodes', color_code_file));
+    if ~isempty(A)
+        detection = [A.V2, A.V3];
+        mat = load(fullfile(results_dir, 'mat', sub_dir_name, mat_file_name));
+        if isfield(mat, 'mat')
+            mat = mat.mat;
+        end
+        cell_ids = mat.cell_ids;
+        C = unique(cell_ids);
+        class = zeros(length(C),1);
+        for j = 1:length(C)
+            class(j) = mode(mat.output(mat.cell_ids==C(j)));
+        end
+        for c = 1:height(colorcodes)
+            image = annotate_image_with_class(image, detection(class==c,:), ...
+                hex2rgb(colorcodes.color{c}), strength);
+        end
+        A.V1 = colorcodes.class(class);
+        writetable(A, fullfile(results_dir, 'csv', sub_dir_name, [mat_file_name(1:end-3), 'csv']));
+    else
+        fileID = fopen(fullfile(results_dir, 'csv', sub_dir_name, [mat_file_name(1:end-3), 'csv']), 'w');
+        fprintf(fileID, 'V1,V2,V3');
+    end
+    mat.class = class;
+    imwrite(image, fullfile(results_dir, 'annotated_images', sub_dir_name, [mat_file_name(1:end-3), 'png']), 'png');
+    parsave_mat(fullfile(results_dir, 'mat', sub_dir_name, mat_file_name),  mat);
+    fprintf('saved %s\n', fullfile(results_dir, 'annotated_images', sub_dir_name, [mat_file_name(1:end-3), 'png']));
+else
+    fprintf('Already saved %s\n', fullfile(results_dir, 'annotated_images', sub_dir_name, [mat_file_name(1:end-3), 'png']));
+end

cell_class/matlab/Temp.m

@@ -0,0 +1 @@
+colorcodes = readtable(fullfile(fileparts(mfilename('fullpath')), 'colorcodes', colorfile));