DataBase.nb

(* Content-type: application/vnd.wolfram.mathematica *)

(*** Wolfram Notebook File ***)
(* http://www.wolfram.com/nb *)

(* CreatedBy='Mathematica 11.1' *)

(*CacheID: 234*)
(* Internal cache information:
NotebookFileLineBreakTest
NotebookFileLineBreakTest
NotebookDataPosition[       158,          7]
NotebookDataLength[     27356,        681]
NotebookOptionsPosition[     23771,        591]
NotebookOutlinePosition[     24327,        610]
CellTagsIndexPosition[     24284,        607]
WindowFrame->Normal*)

(* Beginning of Notebook Content *)
Notebook[{

Cell[CellGroupData[{
Cell[TextData[StyleBox["Making a database from the available arXiv.org \
information", "Section"]], "Section",
 CellChangeTimes->{{3.7068333474790816`*^9, 3.706833356251001*^9}, {
   3.707078392167163*^9, 3.707078398936715*^9}, 3.7071792155283146`*^9, {
   3.7073962287556133`*^9, 3.7073962289927692`*^9}, {3.7074714864433146`*^9, 
   3.7074715004456186`*^9}, {3.707578918721917*^9, 3.707578925525507*^9}, {
   3.707689937323124*^9, 
   3.7076899984675474`*^9}},ExpressionUUID->"d9e11c74-1322-4a03-bae9-\
54415aca75b3"],

Cell[CellGroupData[{

Cell["Names of arXiv\[CloseCurlyQuote]s", "Subsection",
 CellChangeTimes->{{3.7073905110072937`*^9, 3.7073905208607903`*^9}, {
  3.7076921983763885`*^9, 
  3.7076921999605713`*^9}},ExpressionUUID->"498a983d-b919-4830-98c2-\
d5fd5392bee4"],

Cell["\<\
Here we extract the names of the different arXiv categories from the names of \
the folders\
\>", "Text",
 CellChangeTimes->{{3.707605285085425*^9, 3.7076053162387466`*^9}, {
  3.7076055254801035`*^9, 
  3.707605526580083*^9}},ExpressionUUID->"dc54d86f-07c3-4363-99f6-\
09e79a46b8f7"],

Cell[BoxData[
 RowBox[{
  RowBox[{
   RowBox[{
    RowBox[{"SetDirectory", "[", 
     RowBox[{
      RowBox[{"NotebookDirectory", "[", "]"}], "<>", "\"\<Data\\\\9601\>\""}],
      "]"}], ";"}], "\[IndentingNewLine]", 
   RowBox[{
    RowBox[{"arXivNamesList", "=", 
     RowBox[{"FileNames", "[", "]"}]}], ";"}], " ", "\[IndentingNewLine]", 
   RowBox[{
    RowBox[{"currentDirectory", " ", "=", 
     RowBox[{"SetDirectory", "[", 
      RowBox[{"NotebookDirectory", "[", "]"}], "]"}]}], ";"}], 
   "\[IndentingNewLine]", 
   RowBox[{
    RowBox[{"$dataPath", " ", "=", " ", 
     RowBox[{"currentDirectory", "<>", "\"\<\\\\Data\>\""}]}], ";"}]}], " ", 
  "*)"}]], "Input",
 CellChangeTimes->{{3.707390619846746*^9, 3.707390627764739*^9}, {
   3.707470666367057*^9, 3.7074707084430285`*^9}, {3.707470862300887*^9, 
   3.707470882152212*^9}, {3.7074711148838425`*^9, 3.7074712489970107`*^9}, {
   3.707482254590149*^9, 3.707482262046291*^9}, 3.707502201874776*^9, {
   3.70757933980068*^9, 3.707579343948909*^9}, {3.707592737867982*^9, 
   3.7075927383181734`*^9}, {3.7075927894925914`*^9, 3.707592789761295*^9}, {
   3.707692252173673*^9, 3.70769226643633*^9}, {3.707700497609356*^9, 
   3.7077005009935803`*^9}, {3.70784027868233*^9, 3.7078403019709954`*^9}, {
   3.707840645697934*^9, 3.7078406482441883`*^9}, {3.7082203789188566`*^9, 
   3.7082203846221013`*^9}},ExpressionUUID->"b759e4b7-7d38-44d7-91fa-\
dcfaeef75e97"],

Cell[BoxData[
 RowBox[{
  RowBox[{"arXivNamesList", "=", 
   RowBox[{"{", 
    RowBox[{
    "\"\<astro-ph\>\"", ",", "\"\<cond-mat\>\"", ",", "\"\<gr-qc\>\"", ",", 
     "\"\<hep-ex\>\"", ",", "\"\<hep-lat\>\"", ",", "\"\<hep-ph\>\"", ",", 
     "\"\<hep-th\>\"", ",", "\"\<math\>\"", ",", "\"\<math-ph\>\"", ",", 
     "\"\<nucl-ex\>\"", ",", "\"\<nucl-th\>\"", ",", "\"\<physics\>\"", ",", 
     "\"\<quant-ph\>\""}], "}"}]}], ";"}]], "Input",
 CellChangeTimes->{{3.707700510343979*^9, 3.70770053646984*^9}, {
  3.707700670760055*^9, 3.7077006829959145`*^9}, {3.707840308854274*^9, 
  3.7078403115231133`*^9}, {3.707840641764425*^9, 
  3.7078406432979407`*^9}},ExpressionUUID->"3f8fe72a-1595-4aa1-9db6-\
8e1f4db63d6b"]
}, Closed]],

Cell[CellGroupData[{

Cell["Useful functions", "Subsection",
 CellChangeTimes->{{3.707692204738057*^9, 
  3.707692207359044*^9}},ExpressionUUID->"2916f665-d497-4263-a18e-\
e55f01fd7444"],

Cell[BoxData[
 RowBox[{
  RowBox[{
   RowBox[{"yearMonthConvertor", "[", "yearmonthObject_", "]"}], ":=", 
   RowBox[{
    RowBox[{
     RowBox[{"DateString", "[", 
      RowBox[{"yearmonthObject", ",", "#"}], "]"}], "&"}], "/@", 
    RowBox[{"{", 
     RowBox[{"\"\<YearShort\>\"", ",", "\"\<Month\>\""}], "}"}]}]}], 
  ";"}]], "Input",
 CellChangeTimes->{{3.7076922498424673`*^9, 3.707692287421289*^9}, {
  3.7076923490873384`*^9, 3.7076924009193363`*^9}, {3.7076926146738367`*^9, 
  3.70769261962097*^9}},ExpressionUUID->"4af3b372-2208-4bc3-a8bb-\
7e5966c4fb20"]
}, Closed]],

Cell[CellGroupData[{

Cell["Extracting the arXiv numbers", "Subsection",
 CellChangeTimes->{{3.707390647118127*^9, 3.7073906572205143`*^9}, {
  3.707390718372091*^9, 3.707390724811494*^9}, {3.7076942346963015`*^9, 
  3.7076942386292334`*^9}},ExpressionUUID->"7e558412-31ae-4250-8eab-\
4c66ed44d8c7"],

Cell["\<\
We write a few functions to streamline the description of the data paths. We \
take Wolfram Alpha recognizable DateObjects as input and produce a list of \
filenames as the output.\
\>", "Text",
 CellChangeTimes->{{3.707605330742995*^9, 
  3.7076054300794325`*^9}},ExpressionUUID->"799b4d5f-7dee-4461-9a98-\
b9e86f37cda2"],

Cell[BoxData[{
 RowBox[{
  RowBox[{"currentDirectory", " ", "=", 
   RowBox[{"SetDirectory", "[", 
    RowBox[{"NotebookDirectory", "[", "]"}], "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"$dataPath", " ", "=", " ", 
   RowBox[{"currentDirectory", "<>", "\"\<\\\\Data\>\""}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{
   RowBox[{"arXivNumbersList", "[", 
    RowBox[{"year_", ",", "month_", ",", "arXiv_"}], "]"}], ":=", 
   RowBox[{"Module", "[", 
    RowBox[{
     RowBox[{"{", "filenames", "}"}], ",", 
     RowBox[{
      RowBox[{"SetDirectory", "[", 
       RowBox[{"$dataPath", "<>", "\"\<\\\\\>\"", "<>", 
        RowBox[{"(", 
         RowBox[{"StringJoin", "@", 
          RowBox[{"yearMonthConvertor", "[", 
           RowBox[{"DateObject", "[", 
            RowBox[{"{", 
             RowBox[{"year", ",", "month"}], "}"}], "]"}], "]"}]}], ")"}], 
        "<>", "\"\<\\\\\>\"", "<>", "arXiv"}], "]"}], ";", 
      RowBox[{"filenames", "=", 
       RowBox[{"FileNames", "[", "]"}]}], ";", 
      RowBox[{"SetDirectory", "[", 
       RowBox[{"NotebookDirectory", "[", "]"}], "]"}], ";", "filenames"}]}], 
    "]"}]}], ";"}]}], "Input",
 CellChangeTimes->{{3.7074327480672426`*^9, 3.7074327597323675`*^9}, {
  3.707601843194063*^9, 3.7076019256920614`*^9}, {3.707604935430709*^9, 
  3.7076049841229343`*^9}, {3.707606034259755*^9, 3.7076060408603196`*^9}, {
  3.707690356736621*^9, 3.7076904380107946`*^9}, {3.7076905169027834`*^9, 
  3.7076905207124166`*^9}, {3.7076905511069255`*^9, 3.7076905571860247`*^9}, {
  3.7076942265466633`*^9, 
  3.7076942307263556`*^9}},ExpressionUUID->"6960c14f-095f-4717-ad66-\
73b5907fdabc"]
}, Closed]],

Cell[CellGroupData[{

Cell["Extracting the file names", "Subsection",
 CellChangeTimes->{{3.707690708954156*^9, 3.7076907107231846`*^9}, 
   3.7076907982048855`*^9, {3.707693220422695*^9, 3.7076932771171165`*^9}, {
   3.707693317306345*^9, 3.7076933174845085`*^9}, {3.7076933580409155`*^9, 
   3.707693478314123*^9}, {3.7076935141914845`*^9, 3.7076935557766457`*^9}, {
   3.7076936546145587`*^9, 3.707693704506191*^9}, {3.707693839619372*^9, 
   3.707693894004603*^9}, {3.7076939462809772`*^9, 3.7076939801556177`*^9}, {
   3.707694224808428*^9, 
   3.707694251967101*^9}},ExpressionUUID->"f4225acf-fbf7-4bd7-b5ed-\
becb4468eecc"],

Cell[BoxData[
 RowBox[{
  RowBox[{
   RowBox[{"filePaths", "[", 
    RowBox[{"year_", ",", "month_", ",", "arXiv_"}], "]"}], ":=", 
   RowBox[{
    RowBox[{
     RowBox[{
      RowBox[{"FileNames", "[", 
       RowBox[{"\"\<*\>\"", ",", 
        RowBox[{"$dataPath", "<>", "\"\<\\\\\>\"", "<>", 
         RowBox[{"(", 
          RowBox[{"StringJoin", "@", 
           RowBox[{"yearMonthConvertor", "[", 
            RowBox[{"DateObject", "[", 
             RowBox[{"{", 
              RowBox[{"year", ",", "month"}], "}"}], "]"}], "]"}]}], ")"}], 
         "<>", "\"\<\\\\\>\"", "<>", "arXiv", "<>", "\"\<\\\\\>\"", "<>", 
         "#"}]}], "]"}], "\[LeftDoubleBracket]", "1", 
      "\[RightDoubleBracket]"}], "&"}], "/@", 
    RowBox[{"arXivNumbersList", "[", 
     RowBox[{"year", ",", "month", ",", "arXiv"}], "]"}]}]}], ";"}]], "Input",
 CellChangeTimes->{{3.7076942824193096`*^9, 3.7076944584756746`*^9}, 
   3.707694547500868*^9, 3.7076945846797857`*^9, {3.7076946427361555`*^9, 
   3.7076947763733797`*^9}, {3.7076948383991237`*^9, 
   3.7076948778981085`*^9}, {3.707695874708308*^9, 3.707695883940328*^9}, {
   3.707696190081938*^9, 3.70769620498402*^9}, 
   3.707696321903511*^9},ExpressionUUID->"7ef800ad-5f8b-4abd-9a0f-\
e79a3d6e37c0"],

Cell[BoxData[
 RowBox[{
  RowBox[{"(*", " ", 
   RowBox[{"filePaths", "[", 
    RowBox[{"2000", ",", "03", ",", "\"\<astro-ph\>\""}], "]"}], " ", "*)"}], 
  RowBox[{"(*", " ", "Example", " ", "*)"}]}]], "Input",
 CellChangeTimes->{{3.7076943505619216`*^9, 3.707694352246439*^9}, {
   3.7076944637927933`*^9, 3.7076945292608695`*^9}, {3.70769459635104*^9, 
   3.7076946176609893`*^9}, {3.7076947254148474`*^9, 3.707694729631523*^9}, {
   3.707694807165862*^9, 3.707694828995884*^9}, 3.7076948857983303`*^9, {
   3.7076958896410923`*^9, 3.707695895157276*^9}, {3.707696325571867*^9, 
   3.7076963457542887`*^9}},ExpressionUUID->"ee0aeb19-a7e5-4cb3-8979-\
f6493e8e0281"]
}, Closed]],

Cell[CellGroupData[{

Cell["Extracting the relevant symbols in a paper", "Subsection",
 CellChangeTimes->{{3.7076927517218685`*^9, 
  3.707692764121657*^9}},ExpressionUUID->"be852e46-f41d-4123-bced-\
22f4cecaf5c2"],

Cell[BoxData[{
 RowBox[{
  RowBox[{"unwantedSymbols", "=", 
   RowBox[{"FromCharacterCode", "/@", 
    RowBox[{"(", 
     RowBox[{
      RowBox[{"{", 
       RowBox[{
        RowBox[{"Range", "[", "168", "]"}], ",", "8201", ",", "8211", ",", 
        "8217", ",", "8220", ",", "8221", ",", "8242", ",", "8243", ",", 
        "8289", ",", "8290", ",", 
        RowBox[{"Range", "[", 
         RowBox[{"8593", ",", "8601"}], "]"}], ",", "8764", ",", "8805", ",", 
        "8818"}], "}"}], "//", "Flatten"}], ")"}]}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{
   RowBox[{"allSymbols", "[", "article_", "]"}], ":=", 
   RowBox[{"KeyDrop", "[", 
    RowBox[{
     RowBox[{"Counts", "[", 
      RowBox[{"Characters", "[", 
       RowBox[{"Import", "[", 
        RowBox[{"article", ",", "\"\<Text\>\""}], "]"}], "]"}], "]"}], ",", 
     "unwantedSymbols"}], "]"}]}], ";"}]}], "Input",
 CellChangeTimes->{{3.707653105268427*^9, 3.707653165758289*^9}, {
   3.70765325133932*^9, 3.7076532945750155`*^9}, {3.707653347980654*^9, 
   3.70765334949841*^9}, {3.7076534737084017`*^9, 3.707653477055588*^9}, {
   3.707659890696696*^9, 3.70765989960343*^9}, {3.707660007365183*^9, 
   3.707660033583559*^9}, {3.7076600678073597`*^9, 3.7076600689234896`*^9}, {
   3.7076602626631165`*^9, 3.707660263979035*^9}, {3.707660342258831*^9, 
   3.707660344721673*^9}, {3.707660886030304*^9, 3.707660897111148*^9}, {
   3.707683415361715*^9, 3.7076835966506166`*^9}, {3.7076928179027624`*^9, 
   3.707692818149911*^9}, 
   3.7076928494434757`*^9},ExpressionUUID->"869cb8ac-71d9-4c1b-b195-\
dce7e0120b86"],

Cell[BoxData[
 RowBox[{"(*", " ", 
  RowBox[{"ParallelMap", "[", 
   RowBox[{
    RowBox[{
     RowBox[{"allSymbols", "[", "#", "]"}], "&"}], ",", 
    RowBox[{"filePaths", "[", 
     RowBox[{"2000", ",", "1", ",", "\"\<astro-ph\>\""}], "]"}]}], "]"}], " ",
   "*)"}]], "Input",
 CellChangeTimes->{{3.7077500103246517`*^9, 
  3.7077500449457264`*^9}},ExpressionUUID->"582c7ba0-0e8f-41cc-a05c-\
23f1bf8785d2"]
}, Closed]],

Cell[CellGroupData[{

Cell["Extracting the titles of the papers from the XML", "Subsection",
 CellChangeTimes->{{3.7076926818981237`*^9, 
  3.7076926948470974`*^9}},ExpressionUUID->"d8901d3a-0421-4b50-9a49-\
35c3fa0ae809"],

Cell[BoxData[
 RowBox[{
  RowBox[{
   RowBox[{"title", "[", "article_", "]"}], ":=", 
   RowBox[{
    RowBox[{"StringReplace", "[", 
     RowBox[{"\"\<\\n\>\"", "\[Rule]", " ", "\"\< \>\""}], "]"}], "@", 
    RowBox[{"StringJoin", "@", 
     RowBox[{"DeleteCases", "[", 
      RowBox[{
       RowBox[{"Cases", "[", 
        RowBox[{
         RowBox[{"Import", "[", 
          RowBox[{"article", ",", "\"\<XMLObject\>\""}], "]"}], ",", 
         RowBox[{
          RowBox[{"XMLElement", "[", 
           RowBox[{"_", ",", 
            RowBox[{"{", 
             RowBox[{
             "\"\<class\>\"", "\[Rule]", "\"\<title document-title\>\""}], 
             "}"}], ",", 
            RowBox[{"{", "title__", "}"}]}], "]"}], "\[RuleDelayed]", 
          "title"}], ",", "\[Infinity]"}], "]"}], ",", 
       RowBox[{"XMLElement", "[", 
        RowBox[{"_", ",", "_", ",", "_"}], "]"}], ",", "Infinity"}], 
      "]"}]}]}]}], ";"}]], "Input",
 CellChangeTimes->{{3.707691688356189*^9, 3.7076916887044663`*^9}, {
   3.707691766117451*^9, 3.7076919108060365`*^9}, {3.7076919568227034`*^9, 
   3.707691976443203*^9}, 3.707692034924446*^9, {3.707692828387316*^9, 
   3.7076928340097*^9}, 3.707697735975269*^9, {3.707697780165588*^9, 
   3.707697825785006*^9}, {3.7076978826596613`*^9, 3.70769790817686*^9}, {
   3.707697965747691*^9, 3.707697981921173*^9}, {3.7076980184477816`*^9, 
   3.7076980333327713`*^9}, {3.707698724089965*^9, 
   3.7076987326939926`*^9}},ExpressionUUID->"9720861c-f797-406d-b561-\
e03f6f16d07d"],

Cell[BoxData[
 RowBox[{"(*", 
  RowBox[{"ParallelMap", "[", 
   RowBox[{
    RowBox[{
     RowBox[{"title", "[", "#", "]"}], "&"}], ",", 
    RowBox[{"filePaths", "[", 
     RowBox[{"2000", ",", "1", ",", "\"\<astro-ph\>\""}], "]"}]}], "]"}], 
  "*)"}]], "Input",
 CellChangeTimes->{{3.7076981975181684`*^9, 3.7076982414294043`*^9}, {
  3.7076982729899063`*^9, 3.707698279751637*^9}, {3.707698332827447*^9, 
  3.707698334512145*^9}, {3.707698428335683*^9, 3.7076984598191953`*^9}, {
  3.7076986347469535`*^9, 3.7076987291104364`*^9}, {3.707700223189486*^9, 
  3.7077002254566936`*^9}, {3.7077499247486415`*^9, 3.7077499407670045`*^9}, {
  3.7077499838104906`*^9, 
  3.7077499861106215`*^9}},ExpressionUUID->"9f48f96c-a065-42e7-99fb-\
e7d9acd9763b"]
}, Closed]],

Cell[CellGroupData[{

Cell["Association for a single paper", "Subsection",
 CellChangeTimes->{{3.7076949392718143`*^9, 
  3.7076949441723557`*^9}},ExpressionUUID->"1f8777b5-ea43-4a1c-add8-\
0810e42e85ad"],

Cell[BoxData[{
 RowBox[{
  RowBox[{
   RowBox[{"dataAssoc", "[", 
    RowBox[{"year_", ",", "month_", ",", "arXiv_"}], "]"}], ":=", 
   RowBox[{"Module", "[", 
    RowBox[{
     RowBox[{"{", 
      RowBox[{
       RowBox[{"numbers", "=", 
        RowBox[{"UsingFrontEnd", "[", 
         RowBox[{"arXivNumbersList", "[", 
          RowBox[{"year", ",", "month", ",", "arXiv"}], "]"}], "]"}]}], ",", 
       RowBox[{"files", "=", 
        RowBox[{"UsingFrontEnd", "[", 
         RowBox[{"filePaths", "[", 
          RowBox[{"year", ",", "month", ",", "arXiv"}], "]"}], "]"}]}]}], 
      "}"}], ",", "\[IndentingNewLine]", 
     RowBox[{"DeleteCases", "[", 
      RowBox[{
       RowBox[{"Table", "[", 
        RowBox[{
         RowBox[{"AssociationThread", "[", 
          RowBox[{
           RowBox[{"{", 
            RowBox[{
            "\"\<Year\>\"", ",", "\"\<Month\>\"", ",", "\"\<Type\>\"", ",", 
             "\"\<Number\>\"", ",", "\"\<Title\>\"", ",", "\"\<Symbols\>\""}],
             "}"}], ",", 
           RowBox[{"{", 
            RowBox[{"year", ",", "month", ",", "arXiv", ",", 
             RowBox[{
             "numbers", "\[LeftDoubleBracket]", "i", 
              "\[RightDoubleBracket]"}], ",", 
             RowBox[{"title", "[", 
              RowBox[{
              "files", "\[LeftDoubleBracket]", "i", "\[RightDoubleBracket]"}],
               "]"}], ",", 
             RowBox[{"allSymbols", "[", 
              RowBox[{
              "files", "\[LeftDoubleBracket]", "i", "\[RightDoubleBracket]"}],
               "]"}]}], "}"}]}], "]"}], ",", 
         RowBox[{"{", 
          RowBox[{"i", ",", "1", ",", 
           RowBox[{"Length", "[", "numbers", "]"}]}], "}"}]}], "]"}], ",", 
       RowBox[{"<|", 
        RowBox[{"__", ",", 
         RowBox[{"\"\<Title\>\"", "\[Rule]", "\"\<\>\""}], ",", "__"}], 
        "|>"}]}], "]"}]}], "]"}]}], ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{
   RowBox[{"dataAssocallarXivs", "[", 
    RowBox[{"year_", ",", "month_"}], "]"}], ":=", 
   RowBox[{"Flatten", "[", 
    RowBox[{"Map", "[", 
     RowBox[{
      RowBox[{
       RowBox[{"dataAssoc", "[", 
        RowBox[{"year", ",", "month", ",", "#"}], "]"}], "&"}], ",", 
      "arXivNamesList"}], "]"}], "]"}]}], ";"}]}], "Input",
 CellChangeTimes->{{3.707694950059375*^9, 3.7076952511380625`*^9}, {
   3.707695357235752*^9, 3.7076954075255513`*^9}, {3.7076954931786175`*^9, 
   3.707695518555616*^9}, {3.7076963819622164`*^9, 3.7076963929148135`*^9}, {
   3.7076965327376957`*^9, 3.7076965964736032`*^9}, {3.707696631014426*^9, 
   3.707696773625019*^9}, {3.7076968495874963`*^9, 3.707696980283145*^9}, {
   3.707699104842108*^9, 3.7076991103103004`*^9}, {3.707699707244774*^9, 
   3.707699724494029*^9}, {3.7077003698548007`*^9, 3.707700370185937*^9}, {
   3.7077496642718573`*^9, 3.707749683320195*^9}, 3.707749748824008*^9, {
   3.707750347342597*^9, 3.70775034922678*^9}, {3.7077506126365004`*^9, 
   3.707750625416809*^9}, {3.707750655451251*^9, 3.707750688614252*^9}, {
   3.70775087265201*^9, 3.707750873352243*^9}, {3.7077512718912945`*^9, 
   3.7077512774218984`*^9}, 3.707751331315338*^9, {3.707751572559501*^9, 
   3.707751581547792*^9}, {3.707752078159291*^9, 
   3.7077520784748282`*^9}},ExpressionUUID->"ae1b1528-55c0-4e36-965e-\
d684646d9175"]
}, Closed]],

Cell[CellGroupData[{

Cell["Save dataset", "Subsection",
 CellChangeTimes->{{3.707778388989596*^9, 
  3.707778398703499*^9}},ExpressionUUID->"d89740cf-3bd0-469b-8557-\
03c4121a9120"],

Cell[BoxData[
 RowBox[{
  RowBox[{
   RowBox[{"dataset", "[", "year_", "]"}], ":=", 
   RowBox[{"Dataset", "[", 
    RowBox[{"Flatten", "[", 
     RowBox[{"ParallelMap", "[", 
      RowBox[{
       RowBox[{
        RowBox[{"dataAssocallarXivs", "[", 
         RowBox[{"year", ",", "#"}], "]"}], "&"}], ",", 
       RowBox[{"Range", "[", 
        RowBox[{"1", ",", "12"}], "]"}]}], "]"}], "]"}], "]"}]}], 
  ";"}]], "Input",
 CellChangeTimes->{{3.707700278913124*^9, 3.7077003852253647`*^9}, 
   3.7077014273929577`*^9, 3.707701990858102*^9, 3.707702696733283*^9, {
   3.7077034079444265`*^9, 3.7077034081755896`*^9}, {3.707703881370929*^9, 
   3.707703882887261*^9}, 3.70770446988897*^9, {3.7077063395905027`*^9, 
   3.7077064098603888`*^9}, {3.7077497578372364`*^9, 3.707749771838726*^9}, 
   3.707750359510996*^9, {3.707752047513777*^9, 3.707752048793255*^9}, {
   3.707755992016296*^9, 3.7077560001214275`*^9}, {3.7077709883108816`*^9, 
   3.7077709914645557`*^9}, {3.7077743559279547`*^9, 3.707774360295494*^9}, {
   3.7077783460359817`*^9, 3.7077783502494297`*^9}, 3.7077822889315414`*^9, 
   3.707782566503292*^9, {3.707818797242234*^9, 3.7078188002746315`*^9}, {
   3.707823381191065*^9, 3.7078233862899213`*^9}, {3.707834780287508*^9, 
   3.7078347877562265`*^9}, {3.707837886868799*^9, 3.707837891583847*^9}, {
   3.7078401921436567`*^9, 3.7078401950915604`*^9}, {3.7082200076877165`*^9, 
   3.7082200183847466`*^9}},ExpressionUUID->"dbc4f0d4-b6d5-4170-bae8-\
4009961f7c4f"],

Cell["\<\
The datasets are too big to handle together. They were individually exported \
to \[OpenCurlyDoubleQuote].wl\[CloseCurlyDoubleQuote] files and then imported \
to create the full Dataset\
\>", "Text",
 CellChangeTimes->{{3.7082205381861115`*^9, 
  3.708220586405301*^9}},ExpressionUUID->"53a54f5c-8cc7-44e2-94ee-\
4edd0aef4652"],

Cell[BoxData[
 RowBox[{
  RowBox[{"Export", "[", 
   RowBox[{"\"\<Database\\\\data.<>ToString[year]<>.1-12.wl\>\"", ",", 
    RowBox[{"dataset", "[", "year", "]"}]}], "]"}], ";"}]], "Input",
 CellChangeTimes->{{3.7082205934209957`*^9, 
  3.708220597296007*^9}},ExpressionUUID->"2287fdba-7b22-4da0-a633-\
92b51953bf42"],

Cell[BoxData[{
 RowBox[{
  RowBox[{
   RowBox[{"SetDirectory", "[", 
    RowBox[{"NotebookDirectory", "[", "]"}], "]"}], ";"}], 
  " "}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset1998", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.1998.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset1999", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.1999.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset2000", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.2000.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset2001", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.2001.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset2002", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.2002.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset2003", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.2003.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset2004", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.2004.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset2005", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.2005.1-12.wl\>\"", "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{
  RowBox[{"dataset2006", "=", 
   RowBox[{"Import", "[", "\"\<Database\\\\data.2006.1-12.wl\>\"", "]"}]}], 
  ";"}]}], "Input",
 CellChangeTimes->{{3.7082206211400194`*^9, 
  3.70822062510884*^9}},ExpressionUUID->"1cd89a6e-4717-4ad0-832b-\
dbede587cf0d"],

Cell[BoxData[{
 RowBox[{
  RowBox[{"fullDataset", "=", 
   RowBox[{"Dataset", "[", 
    RowBox[{"Flatten", "[", 
     RowBox[{
      RowBox[{
       RowBox[{"(", 
        RowBox[{
         RowBox[{"Symbol", "[", "#", "]"}], "//", "Normal"}], ")"}], "&"}], "/@", 
      RowBox[{"(", 
       RowBox[{
        RowBox[{
         RowBox[{"\"\<dataset\>\"", "<>", 
          RowBox[{"ToString", "[", "#", "]"}]}], "&"}], "/@", 
        RowBox[{"Range", "[", 
         RowBox[{"1998", ",", "2006"}], "]"}]}], ")"}]}], "]"}], "]"}]}], 
  ";"}], "\[IndentingNewLine]", 
 RowBox[{"DumpSave", "[", 
  RowBox[{"\"\<Database\\\\fullDataset.mx\>\"", ",", "fullDataset"}], 
  "]"}]}], "Input",
 CellChangeTimes->{{3.708220275464718*^9, 3.708220300574398*^9}, {
  3.70822043382571*^9, 
  3.7082204404975915`*^9}},ExpressionUUID->"d2b8b0b0-98c2-4beb-a9dd-\
d84c7b54e3e6"],

Cell[BoxData[""], "Input",
 CellChangeTimes->{{3.7077012988787875`*^9, 3.7077013233664846`*^9}, {
   3.707701950868496*^9, 3.7077019532683325`*^9}, {3.7077026842761145`*^9, 
   3.7077027054014397`*^9}, {3.707703537444239*^9, 3.7077035375754733`*^9}, {
   3.7077038939366307`*^9, 3.7077038941683064`*^9}, {3.7077044930055103`*^9, 
   3.7077044934131203`*^9}, {3.707752269027975*^9, 3.7077522745332823`*^9}, {
   3.7077586937060585`*^9, 3.7077587148160343`*^9}, {3.707759734073197*^9, 
   3.707759738725625*^9}, {3.7077710240440845`*^9, 3.7077710291807632`*^9}, {
   3.7077743512486567`*^9, 3.707774363394563*^9}, 3.707778064335974*^9, {
   3.7077783401587725`*^9, 3.707778354846383*^9}, {3.707782569568386*^9, 
   3.707782571238205*^9}, {3.7078188047281*^9, 3.7078188071595025`*^9}, {
   3.707823388421193*^9, 3.7078233906390085`*^9}, {3.707834792220886*^9, 
   3.707834794189331*^9}, {3.707837452344543*^9, 3.7078374530121775`*^9}, {
   3.7078378966017885`*^9, 3.707837900151859*^9}, {3.7078404163817987`*^9, 
   3.707840418686316*^9}, 3.7082199951407695`*^9, {3.708220034921752*^9, 
   3.708220072187168*^9}, 
   3.7082204787792745`*^9},ExpressionUUID->"7e67e27a-f6c7-4808-8dc1-\
4a933a5c83a6"]
}, Closed]]
}, Closed]]
},
WindowSize->{1223, 695},
WindowMargins->{{Automatic, 122}, {Automatic, 31}},
FrontEndVersion->"11.1 for Microsoft Windows (64-bit) (April 18, 2017)",
StyleDefinitions->Notebook[{
   Cell[
    StyleData[StyleDefinitions -> "Default.nb"]]}, Visible -> False, 
  FrontEndVersion -> "11.1 for Microsoft Windows (64-bit) (April 18, 2017)", 
  StyleDefinitions -> "PrivateStylesheetFormatting.nb"]
]
(* End of Notebook Content *)

(* Internal cache information *)
(*CellTagsOutline
CellTagsIndex->{}
*)
(*CellTagsIndex
CellTagsIndex->{}
*)
(*NotebookFileOutline
Notebook[{
Cell[CellGroupData[{
Cell[580, 22, 519, 8, 63, "Section", "ExpressionUUID" -> \
"d9e11c74-1322-4a03-bae9-54415aca75b3"],
Cell[CellGroupData[{
Cell[1124, 34, 238, 4, 43, "Subsection", "ExpressionUUID" -> \
"498a983d-b919-4830-98c2-d5fd5392bee4"],
Cell[1365, 40, 294, 7, 30, "Text", "ExpressionUUID" -> \
"dc54d86f-07c3-4363-99f6-09e79a46b8f7"],
Cell[1662, 49, 1423, 30, 88, "Input", "ExpressionUUID" -> \
"b759e4b7-7d38-44d7-91fa-dcfaeef75e97"],
Cell[3088, 81, 720, 14, 30, "Input", "ExpressionUUID" -> \
"3f8fe72a-1595-4aa1-9db6-8e1f4db63d6b"]
}, Closed]],
Cell[CellGroupData[{
Cell[3845, 100, 164, 3, 35, "Subsection", "ExpressionUUID" -> \
"2916f665-d497-4263-a18e-e55f01fd7444"],
Cell[4012, 105, 565, 14, 30, "Input", "ExpressionUUID" -> \
"4af3b372-2208-4bc3-a8bb-7e5966c4fb20"]
}, Closed]],
Cell[CellGroupData[{
Cell[4614, 124, 277, 4, 35, "Subsection", "ExpressionUUID" -> \
"7e558412-31ae-4250-8eab-4c66ed44d8c7"],
Cell[4894, 130, 332, 7, 49, "Text", "ExpressionUUID" -> \
"799b4d5f-7dee-4461-9a98-b9e86f37cda2"],
Cell[5229, 139, 1665, 39, 145, "Input", "ExpressionUUID" -> \
"6960c14f-095f-4717-ad66-73b5907fdabc"]
}, Closed]],
Cell[CellGroupData[{
Cell[6931, 183, 608, 9, 35, "Subsection", "ExpressionUUID" -> \
"f4225acf-fbf7-4bd7-b5ed-becb4468eecc"],
Cell[7542, 194, 1247, 28, 69, "Input", "ExpressionUUID" -> \
"7ef800ad-5f8b-4abd-9a0f-e79a3d6e37c0"],
Cell[8792, 224, 667, 12, 30, "Input", "ExpressionUUID" -> \
"ee0aeb19-a7e5-4cb3-8979-f6493e8e0281"]
}, Closed]],
Cell[CellGroupData[{
Cell[9496, 241, 192, 3, 35, "Subsection", "ExpressionUUID" -> \
"be852e46-f41d-4123-bced-22f4cecaf5c2"],
Cell[9691, 246, 1592, 35, 50, "Input", "ExpressionUUID" -> \
"869cb8ac-71d9-4c1b-b195-dce7e0120b86"],
Cell[11286, 283, 408, 11, 30, "Input", "ExpressionUUID" -> \
"582c7ba0-0e8f-41cc-a05c-23f1bf8785d2"]
}, Closed]],
Cell[CellGroupData[{
Cell[11731, 299, 200, 3, 35, "Subsection", "ExpressionUUID" -> \
"d8901d3a-0421-4b50-9a49-35c3fa0ae809"],
Cell[11934, 304, 1513, 34, 88, "Input", "ExpressionUUID" -> \
"9720861c-f797-406d-b561-e03f6f16d07d"],
Cell[13450, 340, 748, 16, 30, "Input", "ExpressionUUID" -> \
"9f48f96c-a065-42e7-99fb-e7d9acd9763b"]
}, Closed]],
Cell[CellGroupData[{
Cell[14235, 361, 182, 3, 35, "Subsection", "ExpressionUUID" -> \
"1f8777b5-ea43-4a1c-add8-0810e42e85ad"],
Cell[14420, 366, 3287, 74, 107, "Input", "ExpressionUUID" -> \
"ae1b1528-55c0-4e36-965e-d684646d9175"]
}, Closed]],
Cell[CellGroupData[{
Cell[17744, 445, 160, 3, 35, "Subsection", "ExpressionUUID" -> \
"d89740cf-3bd0-469b-8557-03c4121a9120"],
Cell[17907, 450, 1483, 28, 30, "Input", "ExpressionUUID" -> \
"dbc4f0d4-b6d5-4170-bae8-4009961f7c4f"],
Cell[19393, 480, 337, 7, 30, "Text", "ExpressionUUID" -> \
"53a54f5c-8cc7-44e2-94ee-4edd0aef4652"],
Cell[19733, 489, 318, 7, 30, "Input", "ExpressionUUID" -> \
"2287fdba-7b22-4da0-a633-92b51953bf42"],
Cell[20054, 498, 1634, 44, 202, "Input", "ExpressionUUID" -> \
"1cd89a6e-4717-4ad0-832b-dbede587cf0d"],
Cell[21691, 544, 854, 24, 50, "Input", "ExpressionUUID" -> \
"d2b8b0b0-98c2-4beb-a9dd-d84c7b54e3e6"],
Cell[22548, 570, 1195, 17, 30, "Input", "ExpressionUUID" -> \
"7e67e27a-f6c7-4808-8dc1-4a933a5c83a6"]
}, Closed]]
}, Closed]]
}
]
*)