preprocess.py

# Copyright 2017 Goekcen Eraslan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import pickle, os, numbers

import numpy as np
import scipy as sp
import pandas as pd
import scanpy as sc
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import scale

def read_dataset(adata, transpose=False, test_split=False, copy=False):
    """
    Convenience function to read the dataset and perform basic preprocesing.
    """
    if isinstance(adata, sc.AnnData):
        if copy:
            adata = adata.copy()
    elif isinstance(adata, str):
        adata = sc.read(adata)
    else:
        raise NotImplementedError

    # 处理输入数据：
    # 1. 如果是 AnnData 对象且要求复制，则创建副本
    # 2. 如果是字符串（文件路径），则读取文件
    # 3. 其他类型抛出未实现异常

    # norm_error = 'Make sure that the dataset (adata.X) contains unnormalized count data.'
    # assert 'n_count' not in adata.obs, norm_error

    # if adata.X.size < 50e6: # check if adata.X is integer only if array is small
    #     if sp.sparse.issparse(adata.X):
    #         assert (adata.X.astype(int) != adata.X).nnz == 0, norm_error
    #     else:
    #         assert np.all(adata.X.astype(int) == adata.X), norm_error

    if transpose: adata = adata.transpose()
    # 如果 transpose 为 True，转置数据矩阵

    if test_split:
        #划分测试集 和训练集 划分的是他的的索引值
        train_idx, test_idx = train_test_split(np.arange(adata.n_obs), test_size=0.1, random_state=42)
        #创建一个 pandas Series
        #初始将所有观测标记为 'train'
        # Series 的长度等于总观测数
        spl = pd.Series(['train'] * adata.n_obs)
        #将测试集的索引标记为 'test'
        spl.iloc[test_idx] = 'test'
        adata.obs['DCA_split'] = spl.values
    else:
        adata.obs['DCA_split'] = 'train'

    adata.obs['DCA_split'] = adata.obs['DCA_split'].astype('category')
    print('### Autoencoder: Successfully preprocessed {} genes and {} cells.'.format(adata.n_vars, adata.n_obs))

    return adata


def normalize(adata, filter_min_counts=False, # 是否过滤低计数基因和细胞
                     size_factors=True,  # 是否计算大小因子
                     normalize_input=False, # 是否标准化输入 
                     logtrans_input=False # 是否对输入进行对数转换: 
):

   # 是否对输入进行对数转换
    if filter_min_counts:
         # 过滤计数小于1的基因
        sc.pp.filter_genes(adata, min_counts=1)
        # 过滤计数小于1的细胞
        sc.pp.filter_cells(adata, min_counts=1)

        # 如果需要进行以下任何操作，则保存原始数据的完整副本
    # - 计算大小因子
    # - 标准化输入
    # - 对数转换
    if size_factors or normalize_input or logtrans_input:
        adata.raw = adata.copy()
    else:
        adata.raw = adata

    if size_factors:
        # 按细胞进行标准化
        # 计算大小因子
        # 大小因子 = 每个细胞的计数 / 所有细胞计数的中位数
        sc.pp.normalize_per_cell(adata)
        adata.obs['size_factors'] = adata.obs.n_counts / np.median(adata.obs.n_counts)
    else:
        adata.obs['size_factors'] = 1.0

    if logtrans_input:
        sc.pp.log1p(adata)

    if normalize_input:
        sc.pp.scale(adata)

    return adata