datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Rfam novel sequences with boundary noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Noise =  0\n",
      "Noise =  25\n",
      "Noise =  50\n",
      "Noise =  75\n",
      "Noise =  100\n",
      "Noise =  125\n",
      "Noise =  150\n",
      "Noise =  175\n",
      "Noise =  200\n"
     ]
    }
   ],
   "source": [
    "from utils.seqEncoders import *\n",
    "from utils.ExpConfiguration import *\n",
    "#%run seqEncoders\n",
    "\n",
    "# Run datasets/Rfam-novel/dataset-preparation.R frst to generate fasta files\n",
    "\n",
    "ltrain = get_labels(fastaTrain)\n",
    "np.save('train_labels', ltrain)\n",
    "lval = get_labels(fastaVal)\n",
    "np.save('val_labels', lval)\n",
    "ltest = get_labels(fastaTest)\n",
    "np.save('test_labels', ltest)\n",
    "\n",
    "\n",
    "# Generate sequences with boundary noise ranging among 0,25,50,75,100,..200 percents of sequence length.\n",
    "for bn in bnoise:\n",
    "    print(\"Noise = \", str(bn))\n",
    "    seqTrain = get_seqs_with_bnoise(fastaTrain,nperc=bn)\n",
    "    write_seqs('train_seq_bn'+str(bn)+'.fasta', seqTrain, ltrain)\n",
    "    seqVal = get_seqs_with_bnoise(fastaVal,nperc=bn)\n",
    "    write_seqs('val_seq_bn'+str(bn)+'.fasta', seqVal, lval)\n",
    "    seqTest = get_seqs_with_bnoise(fastaTest,nperc=bn)\n",
    "    write_seqs('test_seq_bn'+str(bn)+'.fasta', seqTest, ltest)       \n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate encoded Rfam novel sequences and applies the corresponding padding schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Padding =  random , Noise =  0 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  0 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  0 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  0 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  0 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  0 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  25 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  25 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  25 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  25 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  25 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  25 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  50 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  50 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  50 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  50 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  50 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  50 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  75 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  75 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  75 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  75 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  75 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  75 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  100 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  100 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  100 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  100 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  100 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  100 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  125 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  125 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  125 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  125 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  125 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  125 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  150 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  150 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  150 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  150 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  150 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  150 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  175 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  175 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  175 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  175 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  175 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  175 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  200 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  200 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  200 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  200 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  200 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  random , Noise =  200 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  0 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  0 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  0 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  0 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  0 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  0 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  25 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  25 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  25 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Padding =  constant , Noise =  25 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  25 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  25 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  50 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  50 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  50 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  50 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  50 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  50 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  75 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  75 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  75 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  75 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  75 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  75 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  100 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  100 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  100 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  100 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  100 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  100 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  125 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  125 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  125 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  125 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  125 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  125 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  150 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  150 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  150 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  150 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  150 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  150 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  175 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  175 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  175 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  175 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  175 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  175 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  200 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  200 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  200 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  200 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  200 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  constant , Noise =  200 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  0 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  0 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  0 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  0 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  0 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  0 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  25 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  25 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  25 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  25 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  25 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  25 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  50 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  50 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  50 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  50 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  50 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  50 , Encoder =  Hilbert\n",
      " Done 105864 total records\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  75 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  75 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  75 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  75 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  75 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  75 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  100 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  100 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  100 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  100 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  100 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  100 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  125 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  125 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  125 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  125 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  125 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  125 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  150 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  150 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  150 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  150 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  150 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  150 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  175 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  175 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  175 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  175 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  175 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  175 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  200 , Encoder =  3mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  200 , Encoder =  2mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  200 , Encoder =  1mer\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  200 , Encoder =  Snake\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  200 , Encoder =  Morton\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n",
      "Padding =  new , Noise =  200 , Encoder =  Hilbert\n",
      " Done 105864 total records\n",
      " Done 17324 total records\n",
      " Done 25342 total records\n"
     ]
    }
   ],
   "source": [
    "from utils.seqEncoders import *\n",
    "from utils.ExpConfiguration import *\n",
    "#%run seqEncoders\n",
    "\n",
    "# For each sequence with boundary noise bn a padding is added and an encoder is applied\n",
    "\n",
    "for p in padds:\n",
    "    for bn in bnoise:\n",
    "        seqTrain = get_seqs_with_bnoise('train_seq_bn'+str(bn)+'.fasta',nperc=0)\n",
    "        seqVal = get_seqs_with_bnoise('val_seq_bn'+str(bn)+'.fasta',nperc=0)\n",
    "        seqTest = get_seqs_with_bnoise('test_seq_bn'+str(bn)+'.fasta',nperc=0)\n",
    "        for en in seqEncoders:\n",
    "            print(\"Padding = \",p,\", Noise = \", str(bn),\", Encoder = \",en['filename'])\n",
    "            train_seq = encode_seqs(seqTrain,enc=en['enc'],encparam=en['param'+str(bn)],padding=p)\n",
    "            val_seq = encode_seqs(seqVal,enc=en['enc'],encparam=en['param'+str(bn)],padding=p)\n",
    "            test_seq = encode_seqs(seqTest,enc=en['enc'],encparam=en['param'+str(bn)],padding=p)\n",
    "            np.save('train_' +  en['filename'] + '_' + p + \"_\" + str(bn) + '_seq', train_seq)\n",
    "            np.save('val_' + en['filename'] + '_' + p + \"_\" + str(bn) +'_seq', val_seq)\n",
    "            np.save('test_' + en['filename'] + '_' + p + \"_\" + str(bn) +'_seq', test_seq)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Random test sequences preserving di-nucleotide freq to test for rejection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t =  1\n",
      "t =  2\n",
      "t =  5\n",
      "t =  10\n"
     ]
    }
   ],
   "source": [
    "from utils.seqEncoders import *\n",
    "from utils.ExpConfiguration import *\n",
    "\n",
    "for nt in t:\n",
    "    print(\"t = \", str(nt))\n",
    "    nfseqTest = get_nonfunctional_seqs(fastaTest,t=nt)\n",
    "    write_seqs('test_rndseq_t'+str(nt)+'.fasta', nfseqTest,['RND']*len(nfseqTest))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate encoded RNAGCN/nRC sequences and applies the corresponding padding schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate sequences from RNAGCN nRC datasets\n",
    "from utils.ExpConfiguration import *\n",
    "from utils.seqEncoders import *\n",
    "# redefine encoders to adapt to new sequence lengths\n",
    "seqEncoders = (\n",
    "               {'enc' : seq2Snake,\n",
    "                'filename' : 'Snake',\n",
    "                'param0' : [[28,28],['A','T','C','G']]},\n",
    "               {'enc' : seq2Hilbert,\n",
    "                'filename' : 'Hilbert',\n",
    "                'param0' : [[32,32],['A','T','C','G']]},\n",
    "               {'enc' : seq2Morton,\n",
    "                'filename' : 'Morton',\n",
    "                'param0' : [[32,32],['A','T','C','G']]},\n",
    "               {'enc' : seq2Kmer,\n",
    "                'filename' : '1mer',\n",
    "                'param0' : [[770],1,['A','T','C','G']]},\n",
    "               {'enc' : seq2Kmer,\n",
    "                'filename' : '2mer',\n",
    "                'param0' : [[385],2,['A','T','C','G']]},\n",
    "               {'enc' : seq2Kmer,\n",
    "                'filename' : '3mer',\n",
    "                'param0' : [[257],3,['A','T','C','G']]}\n",
    "               )\n",
    "\n",
    "fastaTrain='datasets/nRC-public/dataset_nRC_train.fasta'\n",
    "fastaTest='datasets/nRC-public/dataset_nRC_test.fasta'\n",
    "\n",
    "ltrain = get_labels(fastaTrain)\n",
    "np.save('dataset_nRC_train_labels', ltrain)\n",
    "ltest = get_labels(fastaTest)\n",
    "np.save('dataset_nRC_test_labels', ltest)\n",
    "\n",
    "\n",
    "seqTrain = get_seqs_with_bnoise(fastaTrain,nperc=0)\n",
    "write_seqs('dataset_nRC_train_seq.fasta', seqTrain, ltrain)\n",
    "seqTest = get_seqs_with_bnoise(fastaTest,nperc=0)\n",
    "write_seqs('dataset_nRC_test_seq.fasta', seqTest, ltest)       \n",
    "        \n",
    "\n",
    "# Generate encoded sequences with padding among new,constant and random\n",
    "for p in padds:\n",
    "    for en in seqEncoders:\n",
    "        print(\"Padding = \",p,\", Encoder = \",en['filename'])\n",
    "        print(en['filename'],p)\n",
    "        train_seq = encode_seqs(seqTrain,enc=en['enc'],encparam=en['param0'],padding=p)\n",
    "        test_seq = encode_seqs(seqTest,enc=en['enc'],encparam=en['param0'],padding=p)\n",
    "        np.save('dataset_nRC_train_' + en['filename'] + '_' + p + '_seq', train_seq)\n",
    "        np.save('dataset_nRC_test_' + en['filename'] + '_' + p + '_seq', test_seq)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}