Reorganizing.

js/config.js

@@ -0,0 +1,90 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+var scholarius_options = {
+};
+
+var visjs_options = {
+  stabilize: false,
+  nodes: {
+    borderWidth: 2,
+    borderWidthSelected: 2
+  },
+  groups: {
+    standard: {
+      color: {
+        border: '#fff',
+        background: '#fff',
+        highlight: {
+          border: '#fff',
+          background: '#dd3'
+        },
+      },
+    },
+    processing: {
+      color: {
+        border: '#fff',
+        background: '#da2d2f',
+        highlight: {
+          border: '#fff',
+          background: '#dd3'
+        },
+      },
+    },
+
+    leaf: {
+      color: {
+        border: '#fff',
+        background: '#fff',
+        highlight: {
+          border: '#fff',
+          background: '#dd3'
+        },
+      },
+    }
+  },
+  edges: {
+    color: {
+      color:'rgba(255, 255, 255, 0.15)',
+      highlight:'#dd3',
+      hover: '#0000FF'
+    },
+    style: "arrow",
+    width: 1.5,
+    arrowScaleFactor: 0.25
+  },
+  tooltip: {
+    delay: 300,
+    fontColor: "#fff",
+    fontSize: 14,
+    color: {
+      border: "#000",
+      background: "rgba(0,0,0,0.5)"
+    }
+  },
+  physics: {
+    barnesHut: {
+      enabled: true,
+      gravitationalConstant: -2000,
+      centralGravity: 0.2,
+      springLength: 95,
+      springConstant: 0.04,
+      damping: 0.15
+    }
+  }
+};

js/crawler.js

@@ -0,0 +1,179 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function ScholarCrawler(parser, nodes, edges)
+{
+  this.stack = [];
+
+  this.nodes = nodes;
+  this.edges = edges;
+  this.parser = parser;
+
+  this.delay = 1000;
+  this.minimum_citations = 0;
+};
+
+ScholarCrawler.prototype.article_to_node = function(article)
+{
+  return {
+    id: article._id,
+    _id: article._id,
+    article: article,
+    group: 'standard',
+    label: '',
+    shape: 'dot',
+    mass: article.n_citations/2 + 1,
+    radius: 3*Math.pow(article.n_citations, 0.8) + 3,
+    title: "<span class='node_tooltip'>" + article.authors + 
+      ".<a href='' onclick='return open_external(\"" + article.url + "\")' target='_blank'> "
+      + article.title + ".</a> " + article.source + ", " + article.year + ".</span>"
+  };
+};
+
+ScholarCrawler.prototype.add_citations = function(parent_node, levels)
+{
+  assert(levels >= 0, "levels should be non-negative");
+
+  var crawler = this;
+
+  if(parent_node.is_dummy) 
+    return this._add_citations_from_parser(parent_node, levels);
+
+  articles_db.findOne({_id: parent_node._id}, function(err, parent_article_db)
+  {
+    if(parent_article_db === null || !parent_article_db.is_cached)
+      crawler._add_citations_from_parser(parent_node, levels);
+    else
+      crawler._add_citations_from_db(crawler.article_to_node(parent_node.article), levels);
+  });
+}
+
+ScholarCrawler.prototype._add_citations_from_db = function(parent_node, levels)
+{
+  assert(levels >= 0, "levels should be non-negative");
+
+  var crawler = this;
+  citations_db.find({to: parent_node.article._id}, function(err, citations)
+  {
+    citations.forEach(function(citation)
+    {
+      articles_db.findOne({_id: citation.from}, function(err, child_article)
+      {
+        crawler._add_child_article(child_article, parent_node, levels);
+      });
+    });
+
+    parent_node.group = "standard";
+    crawler.nodes.update(parent_node);
+  });
+}
+
+ScholarCrawler.prototype._add_citations_from_parser = function(parent_node, levels)
+{
+  assert(levels >= 0, "levels should be non-negative");
+
+  var crawler = this;
+  this.parser.parse(parent_node.article.citations_url, function(child_articles)
+  {
+    child_articles.forEach(function(child_article)
+    {
+      articles_db.findOne({_id:child_article._id}, function(err, child_article_db)
+      {
+        if(child_article_db === null)
+        {
+          child_article.is_cached = false;
+          articles_db.insert(child_article);
+          crawler._add_child_article(child_article, parent_node, levels);
+        }
+        else
+        {
+          crawler._add_child_article(child_article_db, parent_node, levels);
+        }
+      });
+    });
+
+    if(!parent_node.is_dummy)
+    {
+      parent_node.group = "standard";
+      crawler.nodes.update(parent_node);
+
+      parent_node.article.is_cached = true;
+      articles_db.update({_id: parent_node.article._id}, parent_node.article, {});
+    }
+  });
+}
+
+ScholarCrawler.prototype._add_child_article = function(child_article, parent_node, levels)
+{
+  assert(levels >= 0, "levels should be non-negative");
+
+  var child_node = this.article_to_node(child_article);
+
+  if(child_article.n_citations < this.minimum_citations)
+    return;
+
+  if(!parent_node.is_dummy)
+  {
+    edge = {
+      id: $.md5(child_node._id + parent_node._id),
+      from: child_node._id,
+      to: parent_node._id
+    };
+
+    edge._id = edge.id;
+
+    if(this.edges.getIds().indexOf(edge._id) < 0)
+    {
+      this.edges.add(edge);
+      citations_db.insert(edge);
+    }
+  }
+
+  if(child_article.n_citations > 0 && levels > 0)
+    this.push(child_node, levels-1);
+
+  if(this.nodes.getIds().indexOf(child_node._id) < 0)
+    this.nodes.add(child_node);
+  else
+    this.nodes.update(child_node);
+}
+
+ScholarCrawler.prototype.push = function(parent_node, levels)
+{
+  assert(levels >= 0, "levels should be non-negative");
+
+  this.stack.push([parent_node, levels]);
+  if(!parent_node.is_dummy)
+  {
+    parent_node.group = "processing";
+    if(this.nodes.getIds().indexOf(parent_node._id) >= 0)
+      this.nodes.update(parent_node);
+  }
+};
+
+ScholarCrawler.prototype.start = function()
+{
+  if(this.stack.length > 0)
+  {
+    var args = this.stack.pop();
+    this.add_citations(args[0], args[1]);
+  }
+
+  var crawler = this;
+  setTimeout(function() { crawler.start() }, 1000);
+};

js/main.js

@@ -0,0 +1,45 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+var assert = require('assert');
+var gui  = require('nw.gui');
+var nedb = require('nedb');
+var path = require('path');
+var win = gui.Window.get();
+
+win.maximize();
+
+function createDB(name)
+{
+  return new nedb({  
+    filename : path.join(gui.App.dataPath, name + '.db'),
+    autoload: true
+  });
+}
+
+function open_external(url)
+{
+  gui.Shell.openExternal(url);
+  return false;
+}
+
+var articles_db = createDB("articles");
+var citations_db = createDB("citations");
+var maps_db = createDB("maps");
+
+new MainMenuView().render(document.getElementById("main"));

js/model/map.js

@@ -0,0 +1,26 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function ScholarMap(seed_url)
+{
+  this.edges = [];
+  this.nodes = [];
+  this.seed_url = seed_url;
+  this.is_initialized = false;
+}
+

js/parsers/generic_parser.js

@@ -0,0 +1,33 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function GenericParser()
+{
+    this.scopus_parser = new ScopusParser();
+    this.web_of_science_parser = new WebOfScienceParser();
+}
+
+
+GenericParser.prototype.parse = function(url, callback)
+{
+    if(url.indexOf("scopus.com") >= 0)
+        this.scopus_parser.parse(url, callback);
+        
+    if(url.indexOf("webofknowledge.com") >= 0)
+        this.web_of_science_parser.parse(url, callback);
+}

js/parsers/scopus_parser.js

@@ -0,0 +1,88 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function ScopusParser()
+{
+}
+
+ScopusParser.prototype._parse_results_page = function(url, callback)
+{
+  var iframe = document.createElement("iframe");
+  $(iframe).hide();
+  document.body.appendChild(iframe);
+  iframe.src = url;
+  $(iframe).load(function()
+  {
+    var articles = [];
+    var ibody = $(iframe).contents()[0];
+
+    var select_pages = $(ibody.getElementsByName("resultsPerPage").item(0));
+    if(select_pages.val() != 200)
+    {
+      select_pages.val(200);
+      select_pages.change();
+      return;
+    }
+
+    $(ibody).find(".resultItemLists li").each(function(index, li)
+    {
+      var article = {};
+
+      $(li).find(".docTitle a").each(function(index, tag)
+      {
+        article.url = tag.href;
+        article.title = $(tag).text();
+      });
+
+      $(li).find("a[href*='citedby']").each(function(index, tag)
+      {
+        article.citations_url = tag.href;
+        article.n_citations = parseInt($(tag).text());
+      });
+      
+      $(li).find('.hidden-label').each(function(index, tag)
+      {
+        if($(tag).text().indexOf("Year") == 0)
+          article.year = parseInt($.trim($(tag).next().text()));
+
+        if($(tag).text().indexOf("Authors") == 0)
+          article.authors = $.trim($(tag).next().text());
+
+        if($(tag).text().indexOf("Source") == 0)
+          article.source = $.trim($(tag).next().text());
+      });
+
+      if(!('n_citations' in article)) {
+        article.citations_url = undefined;
+        article.n_citations = 0;
+      }
+      
+      article._id = $.md5(article.title + article.authors);
+      articles.push(article);
+    });
+
+    callback(articles);
+    document.body.removeChild(iframe);
+  });
+}
+
+ScopusParser.prototype.parse = function(url, callback)
+{
+  if(!url) return;
+  this._parse_results_page(url, callback);
+}

js/parsers/web_of_science_parser.js

@@ -0,0 +1,91 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function WebOfScienceParser()
+{
+}
+
+WebOfScienceParser.prototype._parse_results_page = function(url, callback)
+{
+  console.log("parsing");
+  var iframe = document.createElement("iframe");
+  $(iframe).hide();
+  document.body.appendChild(iframe);
+  iframe.src = url;
+
+  $(iframe).load(function()
+  {
+    var articles = [];
+    var ibody = $(iframe).contents()[0];
+
+    var select_pages = $(ibody.getElementsByName("pageSize").item(0));
+    if(select_pages.val() != 50)
+    {
+      select_pages.val(50);
+      select_pages.change();
+      return;
+    }
+
+    $(ibody).find(".search-results-content").each(function(index, li)
+    {
+      var article = {};
+
+      $(li).find("a[href^='/full_record']").each(function(index, tag)
+      {
+        article.url = tag.href;
+        article.title = $.trim($(tag).text());
+      });
+
+      $(li).next().find("a[href^='/CitingArticles']").each(function(index, tag)
+      {
+        article.citations_url = tag.href;
+        article.n_citations = parseInt($(tag).text());
+      });
+      
+      $(li).find('span.label').each(function(index, tag)
+      {
+        if($(tag).text().indexOf("By:") == 0)
+        {
+          article.authors = $.trim($(tag).parent().text()).replace(/^By: /,"");
+          var source_year = $.trim($(tag).parent().next().text());
+          article.source = source_year.replace(/Published.*/, "")
+              .replace(/\s\s+/g, ", ").replace(/, $/, "");
+          article.year = parseInt(source_year.substr(source_year.length-4));
+        }
+      });
+
+      if(!('n_citations' in article))
+      {
+        article.citations_url = undefined;
+        article.n_citations = 0;
+      }
+      
+      article._id = $.md5(article.title + article.authors);
+      articles.push(article);
+    });
+
+    callback(articles);
+    document.body.removeChild(iframe);
+  });
+}
+
+WebOfScienceParser.prototype.parse = function(url, callback)
+{
+  if(!url) return;
+  this._parse_results_page(url, callback);
+}

js/views/main_menu.js

@@ -0,0 +1,60 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function MainMenuView() {};
+
+MainMenuView.prototype.render = function(container)
+{
+  var main_menu_div = document.createElement("div");
+  $(main_menu_div).addClass("main_menu");
+
+  $(main_menu_div).load("app://root/templates/main_menu.html", function()
+  {
+    document.forms[0].onsubmit = function()
+    {
+      var seed_url = $("input[name=url]").val();
+      var view = new ShowMapView(new ScholarMap(seed_url));
+      view.render(container);
+      return false;
+    };
+
+    maps_db.find({}, function(err, maps)
+    {
+        maps.forEach(function(map)
+        {
+            var ul = document.getElementById('saved_maps_container');
+            var li = document.createElement("li");
+			var a = document.createElement("a");
+            a.onclick = function()
+            {
+                new ShowMapView(map).render(container);
+                return false;
+            };
+			a.href = "";
+            $(a).append(map.nodes[0].title);
+			li.appendChild(a);
+            ul.appendChild(li);
+        });
+    });
+    
+  });
+
+  $(container).empty();
+  $(container).removeClass();
+  container.appendChild(main_menu_div);
+}

js/views/show_map.js

@@ -0,0 +1,80 @@
+/* Copyright (C) 2014 Alinson Xavier
+ *
+ * This file is part of Scholarium.
+ *
+ * Scholarium is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this software. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function ShowMapView(map) {
+  this.map = map;
+};
+
+ShowMapView.prototype.render = function(container)
+{
+  this.nodes = new vis.DataSet();
+  this.edges = new vis.DataSet();
+
+  this.nodes.add(this.map.nodes);
+  this.edges.add(this.map.edges);
+
+  document.nodes = this.nodes;
+  document.edges = this.edges;
+
+  var parser = new GenericParser();
+  var crawler = new ScholarCrawler(parser, this.nodes, this.edges);
+  crawler.start();
+
+  document.crawler = crawler;
+
+  if(!this.map.is_initialized)
+  {
+    crawler.push({ is_dummy: true, article: { citations_url: this.map.seed_url } }, 2);
+    this.map.is_initialized = true;
+  }
+
+  var network_div = document.createElement('div');
+  $(network_div).addClass("mynetwork");
+  $(container).empty();
+  container.appendChild(network_div)
+
+  var data = { nodes: this.nodes, edges: this.edges };
+  this.network = new vis.Network(network_div, data, visjs_options);
+
+  this.network.on('doubleClick', function(params)
+  {
+    if(params.nodes.length > 0)
+    {
+      crawler.push(this.nodes.get(params.nodes[0]), 1);
+    }
+    else
+    {
+      console.log("saving map");
+      this.network.storePosition();
+      this.map.nodes = this.nodes.get();
+      this.map.edges = this.edges.get();
+      if(!this.map._id)
+        maps_db.insert(this.map);
+      else
+        maps_db.update({_id: this.map._id}, this.map);
+    }
+  }.bind(this));
+
+  this.network.on("resize", function(params)
+  {
+    var height = $(window).height();
+    var width = $(window).width();
+    $(".mynetwork").css("width", width);
+    $(".mynetwork").css("height", height);
+  });
+};