From a5604a71f938dd5044a6a85a06587ebc75b45565 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 20 Sep 2015 23:02:00 +0200
Subject: [PATCH 01/47] 0.6.0-dev start

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index bb00598..cc2b8a5 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "sourcejs",
   "description": "Living Style Guide Engine for Managing and Developing Front-end Components.",
   "author": "SourceJS",
-  "version": "0.5.6",
+  "version": "0.6.0-dev",
   "license": "MIT",
   "engines": {
     "node": ">=0.10.0 <4.0"

From 63e66055bd8c757bb7b12d668067f99841cb9b67 Mon Sep 17 00:00:00 2001
From: Sergey Zarouski <szarouski@indigo.ca>
Date: Fri, 25 Sep 2015 11:45:41 -0400
Subject: [PATCH 02/47] update middleware docs

---
 docs/api/plugins/readme.md | 45 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/docs/api/plugins/readme.md b/docs/api/plugins/readme.md
index 0d5074e..edd7c17 100644
--- a/docs/api/plugins/readme.md
+++ b/docs/api/plugins/readme.md
@@ -119,11 +119,52 @@ Middlewares are automatically loaded after installation, and are evaluated on ea
 
 ### Modifying Spec contents
 
-We alredy mentioned before that Middlewares use `req.specData.renderedHtml` object to get processed Spec contents and modify them.
+We already mentioned before that Middlewares use `req.specData.renderedHtml` object to get processed Spec contents and modify them.
 
 As any other basic ExpressJS middleware, at the processing start, we get `req` object on input and pass it to the next handler on each request. This architecture allows us to modify `req` contents on each step.
 
-In 0.4.0 middlewares from plugins are connected one by one, sorted by alphabet. In nearest releases we will add a feature for controlling the queue.
+It's possible to define middleware group/order and control group execution sequence. By default all middlewares (and older plugins) will be placed in default group. Here's the default group execution order:
+
+```js
+loadGroupsOrder: [
+  'request',
+  'pre-html',
+  'default',
+  'html',
+  'response'
+]
+```
+
+####Example configurations
+
+In SourceJS core (configurable from `user/options.js`):
+```js
+{
+  core: {
+    middlewares: {
+        list: {
+          md: {
+            enabled: true,
+            order: -1,
+            group: 'pre-html',
+            indexPath: path.join(appRoot, 'core/middlewares/md.js')
+         }
+         'sourcejs-plugin-name': {
+           order: -2
+         }
+      }
+    }
+  }
+}
+```
+Plugin level configuration in `sourcejs-plugin-name/options.js`, will be merged on-top of defaults:
+```js
+module.exports = {
+  order: 1,
+  group: 'pre-html'
+};
+```
+To override core middlewares, changing their order or replacing with alternative plugins, user will need to modify his `options.js` file. For the security reason, plugin level options can't override core middlewares or set an execution order option lower than 0 (zero).
 
 ### Examples
 

From ddcba17540cf6bd8acb9cde43eeb32d735d4723b Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Mon, 28 Sep 2015 08:22:30 +0200
Subject: [PATCH 03/47] #99: initial spec parser version on CHeerion

---
 assets/test/js/tests.js                       |  12 +-
 core/lib/request.js                           |  46 ++++++
 core/lib/specsParser.js                       |  61 ++++++++
 core/middlewares/clarify.js                   | 110 +++++++++-----
 core/routes/redirects.js                      |   3 +-
 options.js                                    |   2 +-
 package.json                                  |   1 -
 test/unit/lib/specParserTest.js               | 139 ++++++++++++++++++
 .../specs/few-examples/index.src.html         |   7 +
 .../partials/specs/few-examples/info.json     |   4 +
 .../unit/partials/specs/nested/index.src.html |  17 +++
 test/unit/partials/specs/nested/info.json     |   4 +
 .../unit/partials/specs/simple/index.src.html |   5 +
 test/unit/partials/specs/simple/info.json     |   4 +
 14 files changed, 372 insertions(+), 43 deletions(-)
 create mode 100644 core/lib/request.js
 create mode 100644 core/lib/specsParser.js
 create mode 100644 test/unit/lib/specParserTest.js
 create mode 100644 test/unit/partials/specs/few-examples/index.src.html
 create mode 100644 test/unit/partials/specs/few-examples/info.json
 create mode 100644 test/unit/partials/specs/nested/index.src.html
 create mode 100644 test/unit/partials/specs/nested/info.json
 create mode 100644 test/unit/partials/specs/simple/index.src.html
 create mode 100644 test/unit/partials/specs/simple/info.json

diff --git a/assets/test/js/tests.js b/assets/test/js/tests.js
index 15c0584..ee9b9d4 100644
--- a/assets/test/js/tests.js
+++ b/assets/test/js/tests.js
@@ -8,9 +8,9 @@ requirejs.config({
     urlArgs: 'cb=' + Math.random(), /* to avoid caching */
 
     paths: {
-        test: '/test',
-        jasmine: '/test/jasmine/jasmine',
-        'jasmine-html': '/test/jasmine/jasmine-html'
+        test: '/jasmine-test',
+        jasmine: '/jasmine-test/jasmine/jasmine',
+        'jasmine-html': '/jasmine-test/jasmine/jasmine-html'
     },
 
     shim: {
@@ -39,9 +39,9 @@ require(['jquery', 'jasmine-html'], function($, jasmine){
 
     var specs = [];
 
-    specs.push('test/spec/moduleSpec');
-    specs.push('test/spec/sectionsSpec');
-    specs.push('test/spec/innerNavigationSpec');
+    specs.push('spec/moduleSpec');
+    specs.push('spec/sectionsSpec');
+    specs.push('spec/innerNavigationSpec');
 
     $(function(){
         require(specs, function(){
diff --git a/core/lib/request.js b/core/lib/request.js
new file mode 100644
index 0000000..ee026e2
--- /dev/null
+++ b/core/lib/request.js
@@ -0,0 +1,46 @@
+var http = require('http');
+var url = require('url');
+var Q = require('q');
+
+module.exports = function(options) {
+    var deferred = Q.defer();
+
+    options = options || {};
+
+    if (!options.path) return;
+
+    var reqOptions = {
+        host: options.host || '127.0.0.1',
+        port: options.port || global.opts.core.server.port,
+        path: options.path + (options.internal ? '?internal=true' : '')
+    };
+    var fullUrl = url.resolve(reqOptions.host + ':' + reqOptions.port, reqOptions.path);
+
+    var reqCallback = function (response) {
+        var endResponse = '';
+
+        response.on('data', function (chunk) {
+            endResponse += chunk;
+        });
+
+        response.on('end', function () {
+            if (typeof options.callback === 'function') {
+                options.callback(endResponse);
+            }
+
+            deferred.resolve(endResponse);
+        });
+    };
+
+    var request = http.request(reqOptions, reqCallback);
+
+    request.on('error', function (e) {
+        deferred.reject({
+            msg: 'Failed loading spec ' + fullUrl
+        });
+    });
+
+    request.end();
+
+    return deferred.promise;
+};
\ No newline at end of file
diff --git a/core/lib/specsParser.js b/core/lib/specsParser.js
new file mode 100644
index 0000000..77dd609
--- /dev/null
+++ b/core/lib/specsParser.js
@@ -0,0 +1,61 @@
+var cheerio = require('cheerio');
+
+module.exports = function (specHTML, sectionID) {
+    var output = {
+        headResources: {},
+        bodyResources: {},
+        contents: []
+    };
+
+    var $ = cheerio.load(specHTML, {decodeEntities: false});
+    var nesting;
+
+    var parseBlock = function (elements, blockID, blockDefaultID) {
+        var headers = ['h2','h3', 'h4'];
+        var currentHeaderLevel = headers[nesting] || '';
+        var nextLevelheading = headers[(nesting + 1)] || '';
+        var untilHook = headers.slice(0, nesting + 2);
+
+        var block = {
+            nested: [],
+            html: []
+        };
+        var nextHeadersArr = elements.filter(nextLevelheading);
+        var $currentScope = elements.first().nextUntil(untilHook.join(', '));
+        $currentScope = elements.first().add($currentScope);
+
+        block.id = blockID;
+        block.visualID = blockDefaultID;
+        block.header = $currentScope.filter(currentHeaderLevel).first().text();
+
+        $currentScope.filter('.source_example').each(function () {
+            block.html.push($(this).html());
+        });
+
+        nesting++;
+
+        // TODO: fix ID's
+        nextHeadersArr.each(function(subIndex){
+            var $nextScope = $(this).nextUntil(nextLevelheading);
+            $nextScope = $(this).first().add($nextScope);
+
+            blockDefaultID = blockDefaultID + '.' +  (subIndex + 1);
+            blockID = $nextScope.filter(nextLevelheading).first().attr('id') || blockDefaultID;
+            block.nested.push(parseBlock($nextScope, blockID, blockDefaultID));
+        });
+
+        return block;
+    };
+
+    $('.source_section').each(function (index) {
+        var $this = $(this);
+        var nextIndexString = (index + 1).toString();
+        var blockID = $this.attr('id') || nextIndexString;
+        var blockDefaultID = nextIndexString;
+        nesting = 0;
+
+        output.contents.push(parseBlock($this.children(), blockID, blockDefaultID));
+    });
+
+    return output;
+};
\ No newline at end of file
diff --git a/core/middlewares/clarify.js b/core/middlewares/clarify.js
index 30ee825..70daac3 100644
--- a/core/middlewares/clarify.js
+++ b/core/middlewares/clarify.js
@@ -5,13 +5,15 @@ var fs = require('fs-extra');
 var url = require('url');
 var Q = require('q');
 var _ = require('lodash');
-var jsdom = require('jsdom');
+var http = require('http');
+var cheerio = require('cheerio');
 
 var ejs = require(path.join(global.pathToApp, 'core/ejsWithHelpers.js'));
 var trackStats = require(path.join(global.pathToApp, 'core/trackStats'));
 var pathToApp = path.dirname(require.main.filename);
 var specUtils = require(path.join(pathToApp, 'core/lib/specUtils'));
 var parseData = require(path.join(pathToApp, 'core/lib/parseData'));
+var specsParser = require(path.join(pathToApp, 'core/lib/specsParser'));
 var htmlParser = require(path.join(pathToApp, 'core/html-tree/html-parser'));
 
 
@@ -103,44 +105,82 @@ var getTplList = function(){
 var parseSpec = function(sections, pathToSpec) {
     var deferred = Q.defer();
 
-    // Parsing spec with JSdom
-    jsdom.env(
-        'http://127.0.0.1:' + global.opts.core.server.port + pathToSpec + '?internal=true',
-        ['http://127.0.0.1:' + global.opts.core.server.port + '/source/assets/js/modules/sectionsParser.js'],
-        function (err, window) {
-            if (err) {
-                deferred.reject({
-                    err: err,
-                    msg: 'JSDOM error'
-                });
-                return;
-            }
+    var options = {
+        host: '127.0.0.1',
+        port: global.opts.core.server.port,
+        path: pathToSpec + '?internal=true'
+    };
+    var fullUrl = url.resolve(options.host + ':' + options.port, options.path);
 
-            var output = {};
+    var callback = function (response) {
+        var specHTML = '';
 
-            var SourceGetSections = window.SourceGetSections;
+        //another chunk of data has been recieved, so append it to `str`
+        response.on('data', function (chunk) {
+            specHTML += chunk;
+        });
 
-            var parser = new SourceGetSections();
-            var allContents = parser.getSpecFull();
+        //the whole response has been recieved, so we just print it out here
+        response.on('end', function () {
+            deferred.resolve({
+                // whole json or by spec
+                output: specsParser(specHTML),
 
-            if (sections) {
-                output = parser.getSpecFull(sections);
-            } else {
-                output = allContents;
-            }
+                // whole json
+                allContents: specsParser(specHTML)
+            });
+        });
+    };
 
-            if (output) {
-                deferred.resolve({
-                    output: output,
-                    allContents: allContents
-                });
-            } else {
-                deferred.reject({
-                    msg: 'Requested sections HTML not found'
-                });
-            }
-        }
-    );
+    var request = http.request(options, callback);
+
+    request.on('error', function (e) {
+        deferred.reject({
+            msg: 'Failed loading spec ' + fullUrl
+        });
+    });
+
+    request.end();
+
+
+    //// Parsing spec with JSdom
+    //jsdom.env(
+    //    'http://127.0.0.1:' + global.opts.core.server.port + pathToSpec + '?internal=true',
+    //    ['http://127.0.0.1:' + global.opts.core.server.port + '/source/assets/js/modules/sectionsParser.js'],
+    //    function (err, window) {
+    //        if (err) {
+    //            deferred.reject({
+    //                err: err,
+    //                msg: 'JSDOM error'
+    //            });
+    //            return;
+    //        }
+    //
+    //        var output = {};
+    //
+    //        var SourceGetSections = window.SourceGetSections;
+    //
+    //        var parser = new SourceGetSections();
+    //        var allContents = parser.getSpecFull();
+    //
+    //        if (sections) {
+    //            output = parser.getSpecFull(sections);
+    //        } else {
+    //            output = allContents;
+    //        }
+    //
+    //        if (output) {
+    //            deferred.resolve({
+    //                output: output,
+    //                allContents: allContents
+    //            });
+    //        } else {
+    //            deferred.reject({
+    //                msg: 'Requested sections HTML not found'
+    //            });
+    //        }
+    //    }
+    //);
 
     return deferred.promise;
 };
@@ -274,6 +314,8 @@ module.exports.process = function(req, res, next) {
             getSpecData(),
             getTplList()
         ]).spread(function(_specData, tplList) {
+            console.log('_specData', JSON.stringify(_specData, null, 4));
+
             var specData = _specData.output;
             var sections = specData.contents ? specData.contents : [];
 
diff --git a/core/routes/redirects.js b/core/routes/redirects.js
index f28a6d8..9e4ae8d 100644
--- a/core/routes/redirects.js
+++ b/core/routes/redirects.js
@@ -35,5 +35,6 @@ global.app.use('/source/assets', express.static(path.join(pathToApp, 'assets')))
 
 // Custom routes
 global.app.use('/docs', express.static(path.join(pathToApp, '/docs')));
+global.app.use('/test', express.static(path.join(pathToApp, '/test')));
 global.app.use('/jsdoc', express.static(path.join(pathToApp, '/jsdoc')));
-global.app.use('/test', express.static(path.join(pathToApp, '/assets/test')));
\ No newline at end of file
+global.app.use('/jasmine-test', express.static(path.join(pathToApp, '/assets/test')));
\ No newline at end of file
diff --git a/options.js b/options.js
index 660bc03..e652798 100644
--- a/options.js
+++ b/options.js
@@ -8,7 +8,7 @@ module.exports = {
         common: {
             defaultLogLevel: 'INFO',
             defaultProdLogLevel: 'ERROR',
-            includedDirs: ['docs'],
+            includedDirs: ['docs', 'test'],
             specPaths: ['specs'],
 
             // Turn on context level setting
diff --git a/package.json b/package.json
index cc2b8a5..7dbff8a 100644
--- a/package.json
+++ b/package.json
@@ -44,7 +44,6 @@
     "grunt-jsdoc": "^0.5.7",
     "grunt-newer": "~0.6.1",
     "gulp-watch": "operatino/gulp-watch#4.3.5-latest-fsevents",
-    "jsdom": "^1.2.1",
     "load-grunt-parent-tasks": "^0.1.1",
     "load-grunt-tasks": "~0.3.0",
     "lodash": "^3.10.1",
diff --git a/test/unit/lib/specParserTest.js b/test/unit/lib/specParserTest.js
new file mode 100644
index 0000000..92f336e
--- /dev/null
+++ b/test/unit/lib/specParserTest.js
@@ -0,0 +1,139 @@
+var path = require('path');
+var expect = require('chai').expect;
+
+var pathToMasterApp = path.resolve('./');
+
+var loadOptions = require(path.join(pathToMasterApp, 'core/loadOptions'));
+global.opts = loadOptions(path.resolve(pathToMasterApp));
+var specsParser = require(path.join(pathToMasterApp, 'core/lib/specsParser'));
+var request = require(path.join(pathToMasterApp, 'core/lib/request'));
+
+describe('specsParser', function () {
+    it('it should return proper object structure', function (done) {
+        var result = specsParser('');
+
+        expect(result).to.be.an('object');
+        expect(result.headResources).to.be.an('object');
+        expect(result.bodyResources).to.be.an('object');
+        expect(result.contents).to.be.an('array');
+
+        done();
+    });
+
+    it('it should return parsed spec object', function (done) {
+        request({
+            path: '/test/unit/partials/specs/simple/',
+            internal: true,
+            callback: function (response) {
+                var result = specsParser(response);
+
+                expect(result).to.deep.equal({
+                    headResources: {},
+                    bodyResources: {},
+                    contents: [{
+                        nested: [],
+                        html: [
+                            'HTML example'
+                        ],
+                        id: 1,
+                        visualID: 1,
+                        header: 'Section header'
+                    }]
+                });
+
+                done();
+            }
+        });
+    });
+
+    it('it should return parsed spec object with few examples', function (done) {
+        request({
+            path: '/test/unit/partials/specs/few-examples/',
+            internal: true,
+            callback: function (response) {
+                var result = specsParser(response);
+
+                expect(result).to.deep.equal({
+                    headResources: {},
+                    bodyResources: {},
+                    contents: [{
+                        nested: [],
+                        html: [
+                            'HTML example 1',
+                            'HTML example 2'
+                        ],
+                        id: 1,
+                        visualID: 1,
+                        header: 'Section header'
+                    }]
+                });
+
+                done();
+            }
+        });
+    });
+
+    it.only('it should properly parse nested IDs', function (done) {
+        request({
+            path: '/test/unit/partials/specs/nested/',
+            internal: true,
+            callback: function (response) {
+                var result = specsParser(response);
+
+                expect(result.contents[0].visualID).to.equal('1');
+                expect(result.contents[0].nested[0].visualID).to.equal('1.1');
+                expect(result.contents[0].nested[1].visualID).to.equal('1.2');
+                expect(result.contents[0].nested[0].nested[0].visualID).to.equal('1.1.1');
+
+                expect(result.contents[0].id).to.equal('1');
+                expect(result.contents[0].nested[0].id).to.equal('custom-sub-id');
+                expect(result.contents[0].nested[1].id).to.equal('1.2');
+                expect(result.contents[0].nested[0].nested[0].id).to.equal('1.1.1');
+
+                done();
+            }
+        });
+    });
+
+    it('it should return nested spec object', function (done) {
+        request({
+            path: '/test/unit/partials/specs/nested/',
+            internal: true,
+            callback: function (response) {
+                var result = specsParser(response);
+
+                expect(result).to.deep.equal({
+                    headResources: {},
+                    bodyResources: {},
+                    contents: [{
+                        html: [
+                            'HTML example 1'
+                        ],
+                        id: '1',
+                        visualID: '1',
+                        header: 'Section header',
+                        nested: [{
+                            nested: [{
+                                nested: [],
+                                html: [
+                                    'HTML example 1.1.1'
+                                ],
+                                id: 'custom-sub-id',
+                                visualID: '1.1.1',
+                                header: 'Section header'
+                            }],
+                            html: [
+                                'HTML example 1.1'
+                            ],
+                            id: '1.1',
+                            visualID: '1.1',
+                            header: 'Section header'
+                        }]
+                    }]
+                });
+
+                done();
+            }
+        });
+    });
+});
\ No newline at end of file
diff --git a/test/unit/partials/specs/few-examples/index.src.html b/test/unit/partials/specs/few-examples/index.src.html
new file mode 100644
index 0000000..e270c3b
--- /dev/null
+++ b/test/unit/partials/specs/few-examples/index.src.html
@@ -0,0 +1,7 @@
+<div class="source_section">
+    <h2>Section header</h2>
+
+    <div class="source_example">HTML example 1</div>
+
+    <div class="source_example">HTML example 2</div>
+</div>
\ No newline at end of file
diff --git a/test/unit/partials/specs/few-examples/info.json b/test/unit/partials/specs/few-examples/info.json
new file mode 100644
index 0000000..5bf65ff
--- /dev/null
+++ b/test/unit/partials/specs/few-examples/info.json
@@ -0,0 +1,4 @@
+{
+    "title": "Few-examples Spec",
+    "template": "doc"
+}
diff --git a/test/unit/partials/specs/nested/index.src.html b/test/unit/partials/specs/nested/index.src.html
new file mode 100644
index 0000000..dad4637
--- /dev/null
+++ b/test/unit/partials/specs/nested/index.src.html
@@ -0,0 +1,17 @@
+<div class="source_section">
+    <h2>Section header</h2>
+
+    <div class="source_example">HTML example 1</div>
+
+    <h3 id="custom-sub-id">Section sub-header</h3>
+
+    <div class="source_example">HTML example 1.1</div>
+
+    <h4>Section sub-sub-header</h4>
+
+    <div class="source_example">HTML example 1.1.1</div>
+
+    <h3>Section sub-header</h3>
+
+    <div class="source_example">HTML example 1.2</div>
+</div>
\ No newline at end of file
diff --git a/test/unit/partials/specs/nested/info.json b/test/unit/partials/specs/nested/info.json
new file mode 100644
index 0000000..071dd19
--- /dev/null
+++ b/test/unit/partials/specs/nested/info.json
@@ -0,0 +1,4 @@
+{
+    "title": "Nested Spec",
+    "template": "doc"
+}
diff --git a/test/unit/partials/specs/simple/index.src.html b/test/unit/partials/specs/simple/index.src.html
new file mode 100644
index 0000000..45ca756
--- /dev/null
+++ b/test/unit/partials/specs/simple/index.src.html
@@ -0,0 +1,5 @@
+<div class="source_section">
+    <h2>Section header</h2>
+
+    <div class="source_example">HTML example</div>
+</div>
\ No newline at end of file
diff --git a/test/unit/partials/specs/simple/info.json b/test/unit/partials/specs/simple/info.json
new file mode 100644
index 0000000..695a230
--- /dev/null
+++ b/test/unit/partials/specs/simple/info.json
@@ -0,0 +1,4 @@
+{
+    "title": "Simple Spec",
+    "template": "doc"
+}

From 21b77d74c2addfac933a79a7298e3ba77a80714b Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 29 Sep 2015 00:11:58 +0200
Subject: [PATCH 04/47] #99: finish section parsing script

---
 core/lib/request.js                           |  2 +
 core/lib/specsParser.js                       | 29 ++++----
 core/middlewares/clarify.js                   |  1 -
 test/unit/lib/specParserTest.js               | 66 ++++++++++++++++---
 .../partials/specs/complex/index.src.html     | 41 ++++++++++++
 test/unit/partials/specs/complex/info.json    |  4 ++
 6 files changed, 120 insertions(+), 23 deletions(-)
 create mode 100644 test/unit/partials/specs/complex/index.src.html
 create mode 100644 test/unit/partials/specs/complex/info.json

diff --git a/core/lib/request.js b/core/lib/request.js
index ee026e2..e82952e 100644
--- a/core/lib/request.js
+++ b/core/lib/request.js
@@ -1,3 +1,5 @@
+'use strict';
+
 var http = require('http');
 var url = require('url');
 var Q = require('q');
diff --git a/core/lib/specsParser.js b/core/lib/specsParser.js
index 77dd609..4d18f90 100644
--- a/core/lib/specsParser.js
+++ b/core/lib/specsParser.js
@@ -1,3 +1,5 @@
+'use strict';
+
 var cheerio = require('cheerio');
 
 module.exports = function (specHTML, sectionID) {
@@ -8,9 +10,10 @@ module.exports = function (specHTML, sectionID) {
     };
 
     var $ = cheerio.load(specHTML, {decodeEntities: false});
-    var nesting;
 
-    var parseBlock = function (elements, blockID, blockDefaultID) {
+    var parseBlock = function (elements, blockID, blockDefaultID, nesting) {
+        nesting = nesting || 0;
+
         var headers = ['h2','h3', 'h4'];
         var currentHeaderLevel = headers[nesting] || '';
         var nextLevelheading = headers[(nesting + 1)] || '';
@@ -26,23 +29,26 @@ module.exports = function (specHTML, sectionID) {
 
         block.id = blockID;
         block.visualID = blockDefaultID;
+
         block.header = $currentScope.filter(currentHeaderLevel).first().text();
 
         $currentScope.filter('.source_example').each(function () {
             block.html.push($(this).html());
         });
 
-        nesting++;
+        if (nextHeadersArr.length > 0) {
+            nesting++;
 
-        // TODO: fix ID's
-        nextHeadersArr.each(function(subIndex){
-            var $nextScope = $(this).nextUntil(nextLevelheading);
-            $nextScope = $(this).first().add($nextScope);
+            nextHeadersArr.each(function(subIndex){
+                var $nextScope = $(this).nextUntil(nextLevelheading);
+                $nextScope = $(this).first().add($nextScope);
 
-            blockDefaultID = blockDefaultID + '.' +  (subIndex + 1);
-            blockID = $nextScope.filter(nextLevelheading).first().attr('id') || blockDefaultID;
-            block.nested.push(parseBlock($nextScope, blockID, blockDefaultID));
-        });
+                var nextBlockDefaultID = blockDefaultID + '.' +  (subIndex + 1);
+                var nextBlockID = $nextScope.filter(nextLevelheading).first().attr('id') || nextBlockDefaultID;
+
+                block.nested.push(parseBlock($nextScope, nextBlockID, nextBlockDefaultID, nesting));
+            });
+        }
 
         return block;
     };
@@ -52,7 +58,6 @@ module.exports = function (specHTML, sectionID) {
         var nextIndexString = (index + 1).toString();
         var blockID = $this.attr('id') || nextIndexString;
         var blockDefaultID = nextIndexString;
-        nesting = 0;
 
         output.contents.push(parseBlock($this.children(), blockID, blockDefaultID));
     });
diff --git a/core/middlewares/clarify.js b/core/middlewares/clarify.js
index 70daac3..2ab4432 100644
--- a/core/middlewares/clarify.js
+++ b/core/middlewares/clarify.js
@@ -6,7 +6,6 @@ var url = require('url');
 var Q = require('q');
 var _ = require('lodash');
 var http = require('http');
-var cheerio = require('cheerio');
 
 var ejs = require(path.join(global.pathToApp, 'core/ejsWithHelpers.js'));
 var trackStats = require(path.join(global.pathToApp, 'core/trackStats'));
diff --git a/test/unit/lib/specParserTest.js b/test/unit/lib/specParserTest.js
index 92f336e..dc96a6d 100644
--- a/test/unit/lib/specParserTest.js
+++ b/test/unit/lib/specParserTest.js
@@ -35,8 +35,8 @@ describe('specsParser', function () {
                         html: [
                             'HTML example'
                         ],
-                        id: 1,
-                        visualID: 1,
+                        id: '1',
+                        visualID: '1',
                         header: 'Section header'
                     }]
                 });
@@ -62,8 +62,8 @@ describe('specsParser', function () {
                             'HTML example 1',
                             'HTML example 2'
                         ],
-                        id: 1,
-                        visualID: 1,
+                        id: '1',
+                        visualID: '1',
                         header: 'Section header'
                     }]
                 });
@@ -73,7 +73,7 @@ describe('specsParser', function () {
         });
     });
 
-    it.only('it should properly parse nested IDs', function (done) {
+    it('it should properly parse nested IDs', function (done) {
         request({
             path: '/test/unit/partials/specs/nested/',
             internal: true,
@@ -95,7 +95,44 @@ describe('specsParser', function () {
         });
     });
 
-    it('it should return nested spec object', function (done) {
+    it('it should properly parse multiple custom ID\'s', function (done) {
+        request({
+            path: '/test/unit/partials/specs/complex/',
+            internal: true,
+            callback: function (response) {
+                var result = specsParser(response);
+
+                expect(result.contents[0].nested[0].id).to.equal('custom-sub-id');
+
+                expect(result.contents[1].id).to.equal('custom-section');
+
+                expect(result.contents[1].nested[0].nested[0].id).to.equal('custom-h4');
+
+                done();
+            }
+        });
+    });
+
+    it('it should properly pass multiple HTML examples', function (done) {
+        request({
+            path: '/test/unit/partials/specs/complex/',
+            internal: true,
+            callback: function (response) {
+                var result = specsParser(response);
+
+                expect(result.contents[0].html[0]).to.equal('HTML example 1');
+                expect(result.contents[1].nested[0].html[0]).to.equal('HTML example 1.1');
+                expect(result.contents[1].nested[0].nested[0].html[0]).to.equal('HTML example 1.1.1');
+                expect(result.contents[1].nested[0].nested[0].html[1]).to.equal('HTML example 1.1.1');
+
+                expect(result.contents[1].nested[1].html[0]).to.equal('HTML example 1.2');
+
+                done();
+            }
+        });
+    });
+
+    it('it should return full nested spec object', function (done) {
         request({
             path: '/test/unit/partials/specs/nested/',
             internal: true,
@@ -118,16 +155,25 @@ describe('specsParser', function () {
                                 html: [
                                     'HTML example 1.1.1'
                                 ],
-                                id: 'custom-sub-id',
+                                id: '1.1.1',
                                 visualID: '1.1.1',
-                                header: 'Section header'
+                                header: 'Section sub-sub-header'
                             }],
                             html: [
                                 'HTML example 1.1'
                             ],
-                            id: '1.1',
+                            id: 'custom-sub-id',
                             visualID: '1.1',
-                            header: 'Section header'
+                            header: 'Section sub-header'
+                        },
+                        {
+                            nested: [],
+                            html: [
+                                'HTML example 1.2'
+                            ],
+                            id: '1.2',
+                            visualID: '1.2',
+                            header: 'Section sub-header'
                         }]
                     }]
                 });
diff --git a/test/unit/partials/specs/complex/index.src.html b/test/unit/partials/specs/complex/index.src.html
new file mode 100644
index 0000000..fe63052
--- /dev/null
+++ b/test/unit/partials/specs/complex/index.src.html
@@ -0,0 +1,41 @@
+<div class="source_section">
+    <h2>Section header</h2>
+
+    <div class="source_example">HTML example 1</div>
+
+    <h3 id="custom-sub-id">Section sub-header</h3>
+
+    <div class="source_example">HTML example 1.1</div>
+
+    <h4>Section sub-sub-header</h4>
+
+    <div class="source_example">HTML example 1.1.1</div>
+
+    <h3>Section sub-header</h3>
+
+    <div class="source_example">HTML example 1.2</div>
+</div>
+
+<div class="source_section" id="custom-section">
+    <h2 id="custom-section-false">Section header</h2>
+
+    <div class="source_example">
+        <h1>Example header</h1>
+        <h2>Example header</h2>
+        <h3>Example header</h3>
+        <h4>Example header</h4>
+    </div>
+
+    <h3>Section sub-header</h3>
+
+    <div class="source_example">HTML example 1.1</div>
+
+    <h4 id="custom-h4">Section sub-sub-header</h4>
+
+    <div class="source_example">HTML example 1.1.1</div>
+    <div class="source_example">HTML example 1.1.1</div>
+
+    <h3>Section sub-header</h3>
+
+    <div class="source_example">HTML example 1.2</div>
+</div>
\ No newline at end of file
diff --git a/test/unit/partials/specs/complex/info.json b/test/unit/partials/specs/complex/info.json
new file mode 100644
index 0000000..071dd19
--- /dev/null
+++ b/test/unit/partials/specs/complex/info.json
@@ -0,0 +1,4 @@
+{
+    "title": "Nested Spec",
+    "template": "doc"
+}

From ff8429b54053e84d2be5665ac4b65639a3775a34 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 29 Sep 2015 01:27:26 +0200
Subject: [PATCH 05/47] #99: working clarify with cheerio, and selection by
 section

---
 core/lib/parseData.js                         | 40 +++++++----
 .../lib/{specsParser.js => specPageParser.js} | 24 ++++++-
 core/middlewares/clarify.js                   | 69 +++++--------------
 package.json                                  |  3 +-
 ...pecParserTest.js => specPageParserSpec.js} | 42 ++++++++---
 test/unit/lib/{utils.js => utilsSpec.js}      |  0
 .../clarify.js => middlewares/clarifySpec.js} |  6 +-
 .../md.js => middlewares/mdSpec.js}           |  0
 8 files changed, 106 insertions(+), 78 deletions(-)
 rename core/lib/{specsParser.js => specPageParser.js} (75%)
 rename test/unit/lib/{specParserTest.js => specPageParserSpec.js} (82%)
 rename test/unit/lib/{utils.js => utilsSpec.js} (100%)
 rename test/unit/{middleware/clarify.js => middlewares/clarifySpec.js} (98%)
 rename test/unit/{middleware/md.js => middlewares/mdSpec.js} (100%)

diff --git a/core/lib/parseData.js b/core/lib/parseData.js
index b68c0ce..8b731bc 100644
--- a/core/lib/parseData.js
+++ b/core/lib/parseData.js
@@ -15,6 +15,8 @@ var flattenTillSpec = require(path.join(global.pathToApp, 'core/lib/flattenTillS
  * @param {String} config.path - path do data
  */
 function ParseData(config) {
+    config = config || {};
+
     this.data = {};
     this.scope = config.scope;
     this.dataPath = config.path;
@@ -292,26 +294,36 @@ ParseData.prototype.flattenHTMLcontents = function(contents) {
 ParseData.prototype.getBySection = function(id, sections) {
     // Sections are defined only in html data storage
     if (this.scope === 'html' && Array.isArray(sections) && sections.length > 0) {
-        var specData = this.getByID(id);
+        return this.traverseSections(this.getByID(id), sections);
+    } else {
+        return undefined;
+    }
+};
 
-        if (specData) {
-            var specSections = this.flattenHTMLcontents(specData.contents);
+/**
+ * Traverse Spec page object sections
+ *
+ * @param {Object} specData - HTML data structure to parse
+ * @param {Array} sections - Array of sections to return
+ *
+ * @returns {Object} Return single object by requested ID with specified sections HTML OR undefined
+ */
+ParseData.prototype.traverseSections = function(specData, sections) {
+    if (specData && sections) {
+        var specSections = this.flattenHTMLcontents(specData.contents);
 
-            var pickedSections = [];
+        var pickedSections = [];
 
-            sections.forEach(function(id){
-                var objectToAdd = specSections[id];
+        sections.forEach(function(sectionID){
+            var objectToAdd = specSections[sectionID];
 
-                if (objectToAdd) pickedSections.push(objectToAdd);
-            });
+            if (objectToAdd) pickedSections.push(objectToAdd);
+        });
 
-            if (pickedSections.length !== 0) {
-                specData.contents = pickedSections;
+        if (pickedSections.length > 0) {
+            specData.contents = pickedSections;
 
-                return specData;
-            } else {
-                return undefined;
-            }
+            return specData;
         } else {
             return undefined;
         }
diff --git a/core/lib/specsParser.js b/core/lib/specPageParser.js
similarity index 75%
rename from core/lib/specsParser.js
rename to core/lib/specPageParser.js
index 4d18f90..097737f 100644
--- a/core/lib/specsParser.js
+++ b/core/lib/specPageParser.js
@@ -1,8 +1,16 @@
 'use strict';
 
 var cheerio = require('cheerio');
-
-module.exports = function (specHTML, sectionID) {
+var ParseData = new (require('./parseData'));
+
+/**
+ * Parse Spec page contents into HTML tree
+ *
+ * @param {String} specHTML - Full Spec page html (from request)
+ *
+ * @returns {Object} Return parsed Spec page object
+ */
+var process = module.exports.process = function (specHTML) {
     var output = {
         headResources: {},
         bodyResources: {},
@@ -63,4 +71,16 @@ module.exports = function (specHTML, sectionID) {
     });
 
     return output;
+};
+
+/**
+ * Get specific sections from parsed Spec page object
+ *
+ * @param {Object} specObject - Parsed Spec page object
+ * @param {Array} sections - Array of sections to return
+ *
+ * @returns {Object} Return object with specified sections HTML OR undefined
+ */
+module.exports.getBySection = function (specObject, sections) {
+    return ParseData.traverseSections(specObject, sections);
 };
\ No newline at end of file
diff --git a/core/middlewares/clarify.js b/core/middlewares/clarify.js
index 2ab4432..308fa81 100644
--- a/core/middlewares/clarify.js
+++ b/core/middlewares/clarify.js
@@ -12,7 +12,7 @@ var trackStats = require(path.join(global.pathToApp, 'core/trackStats'));
 var pathToApp = path.dirname(require.main.filename);
 var specUtils = require(path.join(pathToApp, 'core/lib/specUtils'));
 var parseData = require(path.join(pathToApp, 'core/lib/parseData'));
-var specsParser = require(path.join(pathToApp, 'core/lib/specsParser'));
+var specsParser = require(path.join(pathToApp, 'core/lib/specPageParser'));
 var htmlParser = require(path.join(pathToApp, 'core/html-tree/html-parser'));
 
 
@@ -100,7 +100,6 @@ var getTplList = function(){
     return deferred.promise;
 };
 
-// TODO: Move to standalone API, for fast JSDOM spec parsing
 var parseSpec = function(sections, pathToSpec) {
     var deferred = Q.defer();
 
@@ -121,13 +120,25 @@ var parseSpec = function(sections, pathToSpec) {
 
         //the whole response has been recieved, so we just print it out here
         response.on('end', function () {
-            deferred.resolve({
-                // whole json or by spec
-                output: specsParser(specHTML),
+            var output = {};
+            var allContents = specsParser.process(specHTML);
 
-                // whole json
-                allContents: specsParser(specHTML)
-            });
+            if (sections) {
+                output = specsParser.getBySection(allContents, sections);
+            } else {
+                output = allContents;
+            }
+
+            if (output) {
+                deferred.resolve({
+                    output: output,
+                    allContents: allContents
+                });
+            } else {
+                deferred.reject({
+                    msg: 'Requested sections HTML not found'
+                });
+            }
         });
     };
 
@@ -141,46 +152,6 @@ var parseSpec = function(sections, pathToSpec) {
 
     request.end();
 
-
-    //// Parsing spec with JSdom
-    //jsdom.env(
-    //    'http://127.0.0.1:' + global.opts.core.server.port + pathToSpec + '?internal=true',
-    //    ['http://127.0.0.1:' + global.opts.core.server.port + '/source/assets/js/modules/sectionsParser.js'],
-    //    function (err, window) {
-    //        if (err) {
-    //            deferred.reject({
-    //                err: err,
-    //                msg: 'JSDOM error'
-    //            });
-    //            return;
-    //        }
-    //
-    //        var output = {};
-    //
-    //        var SourceGetSections = window.SourceGetSections;
-    //
-    //        var parser = new SourceGetSections();
-    //        var allContents = parser.getSpecFull();
-    //
-    //        if (sections) {
-    //            output = parser.getSpecFull(sections);
-    //        } else {
-    //            output = allContents;
-    //        }
-    //
-    //        if (output) {
-    //            deferred.resolve({
-    //                output: output,
-    //                allContents: allContents
-    //            });
-    //        } else {
-    //            deferred.reject({
-    //                msg: 'Requested sections HTML not found'
-    //            });
-    //        }
-    //    }
-    //);
-
     return deferred.promise;
 };
 
@@ -313,8 +284,6 @@ module.exports.process = function(req, res, next) {
             getSpecData(),
             getTplList()
         ]).spread(function(_specData, tplList) {
-            console.log('_specData', JSON.stringify(_specData, null, 4));
-
             var specData = _specData.output;
             var sections = specData.contents ? specData.contents : [];
 
diff --git a/package.json b/package.json
index 7dbff8a..f25779d 100644
--- a/package.json
+++ b/package.json
@@ -58,7 +58,8 @@
     "shortid": "^2.2.2",
     "time-grunt": "~0.2.10",
     "tinyforever": "0.0.3",
-    "universal-analytics": "^0.3.9"
+    "universal-analytics": "^0.3.9",
+    "jsdom": "^1.2.1"
   },
   "scripts": {
     "postinstall": "node ./core/postInstall && grunt update && node ./core/trackInstall",
diff --git a/test/unit/lib/specParserTest.js b/test/unit/lib/specPageParserSpec.js
similarity index 82%
rename from test/unit/lib/specParserTest.js
rename to test/unit/lib/specPageParserSpec.js
index dc96a6d..c263b96 100644
--- a/test/unit/lib/specParserTest.js
+++ b/test/unit/lib/specPageParserSpec.js
@@ -5,12 +5,12 @@ var pathToMasterApp = path.resolve('./');
 
 var loadOptions = require(path.join(pathToMasterApp, 'core/loadOptions'));
 global.opts = loadOptions(path.resolve(pathToMasterApp));
-var specsParser = require(path.join(pathToMasterApp, 'core/lib/specsParser'));
+var specsParser = require(path.join(pathToMasterApp, 'core/lib/specPageParser'));
 var request = require(path.join(pathToMasterApp, 'core/lib/request'));
 
-describe('specsParser', function () {
+describe('specsParser.process', function () {
     it('it should return proper object structure', function (done) {
-        var result = specsParser('');
+        var result = specsParser.process('');
 
         expect(result).to.be.an('object');
         expect(result.headResources).to.be.an('object');
@@ -25,7 +25,7 @@ describe('specsParser', function () {
             path: '/test/unit/partials/specs/simple/',
             internal: true,
             callback: function (response) {
-                var result = specsParser(response);
+                var result = specsParser.process(response);
 
                 expect(result).to.deep.equal({
                     headResources: {},
@@ -51,7 +51,7 @@ describe('specsParser', function () {
             path: '/test/unit/partials/specs/few-examples/',
             internal: true,
             callback: function (response) {
-                var result = specsParser(response);
+                var result = specsParser.process(response);
 
                 expect(result).to.deep.equal({
                     headResources: {},
@@ -78,7 +78,7 @@ describe('specsParser', function () {
             path: '/test/unit/partials/specs/nested/',
             internal: true,
             callback: function (response) {
-                var result = specsParser(response);
+                var result = specsParser.process(response);
 
                 expect(result.contents[0].visualID).to.equal('1');
                 expect(result.contents[0].nested[0].visualID).to.equal('1.1');
@@ -100,7 +100,7 @@ describe('specsParser', function () {
             path: '/test/unit/partials/specs/complex/',
             internal: true,
             callback: function (response) {
-                var result = specsParser(response);
+                var result = specsParser.process(response);
 
                 expect(result.contents[0].nested[0].id).to.equal('custom-sub-id');
 
@@ -118,7 +118,7 @@ describe('specsParser', function () {
             path: '/test/unit/partials/specs/complex/',
             internal: true,
             callback: function (response) {
-                var result = specsParser(response);
+                var result = specsParser.process(response);
 
                 expect(result.contents[0].html[0]).to.equal('HTML example 1');
                 expect(result.contents[1].nested[0].html[0]).to.equal('HTML example 1.1');
@@ -137,7 +137,7 @@ describe('specsParser', function () {
             path: '/test/unit/partials/specs/nested/',
             internal: true,
             callback: function (response) {
-                var result = specsParser(response);
+                var result = specsParser.process(response);
 
                 expect(result).to.deep.equal({
                     headResources: {},
@@ -178,6 +178,30 @@ describe('specsParser', function () {
                     }]
                 });
 
+                done();
+            }
+        });
+    });
+});
+
+describe('specsParser.getBySection', function () {
+    it('it should properly get by single section', function (done) {
+        request({
+            path: '/test/unit/partials/specs/nested/',
+            internal: true,
+            callback: function (response) {
+                var result = specsParser.process(response);
+                var assertOption = function (sections) {
+                    var endResult = specsParser.getBySection(result, sections);
+
+                    expect(endResult.contents.length).to.equal(1);
+                    expect(endResult.contents[0].id).to.equal(sections[0]);
+                };
+
+                assertOption(['1']);
+                assertOption(['custom-sub-id']);
+                assertOption(['1.1.1']);
+
                 done();
             }
         });
diff --git a/test/unit/lib/utils.js b/test/unit/lib/utilsSpec.js
similarity index 100%
rename from test/unit/lib/utils.js
rename to test/unit/lib/utilsSpec.js
diff --git a/test/unit/middleware/clarify.js b/test/unit/middlewares/clarifySpec.js
similarity index 98%
rename from test/unit/middleware/clarify.js
rename to test/unit/middlewares/clarifySpec.js
index eafd6c5..92e02cb 100644
--- a/test/unit/middleware/clarify.js
+++ b/test/unit/middlewares/clarifySpec.js
@@ -7,6 +7,8 @@ var jq = fs.readFileSync(path.join(pathToMasterApp,'assets/js/lib/jquery-2.1.4.m
 var should = require('should');
 var assert = require('assert');
 var request = require('supertest');
+
+// TODO: rewrite tests with cheerio
 var jsdom = require('jsdom');
 
 var loadOptions = require(path.join(pathToMasterApp, 'core/loadOptions'));
@@ -19,7 +21,7 @@ describe('Clarify test /docs/spec?clarify=true', function () {
         it('should return nothing (&sections=77)', function (done) {
             request(url)
                 .get('&sections=77')
-                .expect(200)
+                .expect(500)
                 .end(function (err, res) {
                     if (err) {
                         throw err;
@@ -90,7 +92,7 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                 });
         });
 
-        it('should have injected resources', function (done) {
+        xit('should have injected resources', function (done) {
             request(url)
                 .get('&sections=1.1')
                 .expect(200)
diff --git a/test/unit/middleware/md.js b/test/unit/middlewares/mdSpec.js
similarity index 100%
rename from test/unit/middleware/md.js
rename to test/unit/middlewares/mdSpec.js

From bf7847430d640ac3e50eedc7ea816facd9e302ca Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 29 Sep 2015 01:34:21 +0200
Subject: [PATCH 06/47] fix linting issues

---
 core/lib/specPageParser.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/lib/specPageParser.js b/core/lib/specPageParser.js
index 097737f..cbb13b9 100644
--- a/core/lib/specPageParser.js
+++ b/core/lib/specPageParser.js
@@ -1,7 +1,7 @@
 'use strict';
 
 var cheerio = require('cheerio');
-var ParseData = new (require('./parseData'));
+var ParseData = new (require('./parseData'))();
 
 /**
  * Parse Spec page contents into HTML tree
@@ -10,7 +10,7 @@ var ParseData = new (require('./parseData'));
  *
  * @returns {Object} Return parsed Spec page object
  */
-var process = module.exports.process = function (specHTML) {
+module.exports.process = function (specHTML) {
     var output = {
         headResources: {},
         bodyResources: {},

From 23e433d3bf763e84bd357544bd288ec68a0bda07 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 16:49:38 +0200
Subject: [PATCH 07/47] fix: custom view resolving in

---
 Gruntfile.js             |  4 ++--
 core/middlewares/wrap.js | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/Gruntfile.js b/Gruntfile.js
index 727399a..5fca981 100755
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -328,7 +328,7 @@ module.exports = function(grunt) {
 
     // Test task. Execute with running app
     grunt.registerTask('test', 'Run ALL tests or specified by second param', function () {
-        // if custom mask set - `grunt test --spec=test/unit/middleware/**/*.js`
+        // if custom mask set - `grunt test --spec=test/unit/**/*.js`
         var spec = grunt.option('spec');
         if (spec) {
             grunt.config.set('mochaTest.test.src', [spec]);
@@ -340,7 +340,7 @@ module.exports = function(grunt) {
 
     // Test task. Execute with running app
     grunt.registerTask('test-func', 'Run ALL functional tests or specified by second param', function () {
-        // if custom mask set - `grunt test --spec=test/unit/middleware/**/*.js`
+        // if custom mask set - `grunt test --spec=test/unit/**/*.js`
         var spec = grunt.option('spec');
         if (spec) {
             grunt.config.set('casperjs.files', [spec]);
diff --git a/core/middlewares/wrap.js b/core/middlewares/wrap.js
index 075d0c2..99d66e5 100644
--- a/core/middlewares/wrap.js
+++ b/core/middlewares/wrap.js
@@ -40,6 +40,17 @@ exports.process = function (req, res, next) {
             viewParam = 'navigation';
         }
 
+        // Check if we're working with included dirs
+        if (context) {
+            global.opts.core.common.includedDirs.forEach(function (includedDir) {
+                var pathToCheck = path.join(global.app.get('user'), includedDir) + '/';
+
+                if (context.indexOf(pathToCheck) === 0) {
+                    context = context.replace(pathToCheck, path.join(global.pathToApp, includedDir) + '/')
+                }
+            });
+        }
+
         var templatePath = viewResolver(viewParam, contextOptions.rendering.views, context) || viewParam;
 
         fs.readFile(templatePath, "utf-8", function(err, template){

From e19bfad048aca827ac431f1247aeb0de6ba045d0 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 16:53:08 +0200
Subject: [PATCH 08/47] add reserved notice

---
 test/index.src.html | 5 +++++
 test/info.json      | 5 +++++
 2 files changed, 10 insertions(+)
 create mode 100644 test/index.src.html
 create mode 100644 test/info.json

diff --git a/test/index.src.html b/test/index.src.html
new file mode 100644
index 0000000..426541b
--- /dev/null
+++ b/test/index.src.html
@@ -0,0 +1,5 @@
+<div class="source_subhead">
+    <h1>Reserved Catalog</h1>
+</div>
+
+<p>This specs catalogue path is reserved for core features.</p>
\ No newline at end of file
diff --git a/test/info.json b/test/info.json
new file mode 100644
index 0000000..e60890b
--- /dev/null
+++ b/test/info.json
@@ -0,0 +1,5 @@
+{
+    "title": "Reserved Catalog",
+    "role": "navigation",
+    "tag": ["hidden"]
+}

From d0660552619e8e3e9a87be9efd0a8e1fe428ad8b Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 18:07:06 +0200
Subject: [PATCH 09/47] #99: add resource parsing and tests, move partials to
 tests root folder

---
 core/lib/specPageParser.js                    | 134 +++++++-
 .../partials/markdown-ejs-nested.md           |   0
 test/{unit => }/partials/markdown-ejs.md      |   0
 test/{unit => }/partials/markdown.md          |   0
 test/{unit => }/partials/mask-one.html        |   0
 test/{unit => }/partials/mask-two.html        |   0
 .../partials/specs/complex/index.src.html     |   0
 test/partials/specs/complex/info.json         |   5 +
 .../specs/few-examples/index.src.html         |   0
 test/partials/specs/few-examples/info.json    |   5 +
 .../specs/many-assets}/index.src.html         |   0
 test/partials/specs/many-assets/info.json     |   5 +
 test/partials/specs/many-assets/template.ejs  |  48 +++
 .../partials/specs/nested/index.src.html      |   0
 test/partials/specs/nested/info.json          |   5 +
 test/partials/specs/simple/index.src.html     |   5 +
 test/partials/specs/simple/info.json          |   5 +
 test/{unit => }/partials/three.html           |   0
 test/unit/ejsHelpersSpec.js                   |  10 +-
 test/unit/lib/specPageParserSpec.js           | 318 +++++++++++++++---
 test/unit/middlewares/clarifySpec.js          |   2 +-
 test/unit/partials/specs/complex/info.json    |   4 -
 .../partials/specs/few-examples/info.json     |   4 -
 test/unit/partials/specs/nested/info.json     |   4 -
 test/unit/partials/specs/simple/info.json     |   4 -
 25 files changed, 492 insertions(+), 66 deletions(-)
 rename test/{unit => }/partials/markdown-ejs-nested.md (100%)
 rename test/{unit => }/partials/markdown-ejs.md (100%)
 rename test/{unit => }/partials/markdown.md (100%)
 rename test/{unit => }/partials/mask-one.html (100%)
 rename test/{unit => }/partials/mask-two.html (100%)
 rename test/{unit => }/partials/specs/complex/index.src.html (100%)
 create mode 100644 test/partials/specs/complex/info.json
 rename test/{unit => }/partials/specs/few-examples/index.src.html (100%)
 create mode 100644 test/partials/specs/few-examples/info.json
 rename test/{unit/partials/specs/simple => partials/specs/many-assets}/index.src.html (100%)
 create mode 100644 test/partials/specs/many-assets/info.json
 create mode 100644 test/partials/specs/many-assets/template.ejs
 rename test/{unit => }/partials/specs/nested/index.src.html (100%)
 create mode 100644 test/partials/specs/nested/info.json
 create mode 100644 test/partials/specs/simple/index.src.html
 create mode 100644 test/partials/specs/simple/info.json
 rename test/{unit => }/partials/three.html (100%)
 delete mode 100644 test/unit/partials/specs/complex/info.json
 delete mode 100644 test/unit/partials/specs/few-examples/info.json
 delete mode 100644 test/unit/partials/specs/nested/info.json
 delete mode 100644 test/unit/partials/specs/simple/info.json

diff --git a/core/lib/specPageParser.js b/core/lib/specPageParser.js
index cbb13b9..d76c0c0 100644
--- a/core/lib/specPageParser.js
+++ b/core/lib/specPageParser.js
@@ -3,22 +3,144 @@
 var cheerio = require('cheerio');
 var ParseData = new (require('./parseData'))();
 
+/**
+ * Filter list of HTML nodes
+ *
+ * @param {Object} $ - cheerio root object
+ * @param {Array} $elements - nodelist to filter
+ * @param {Function} [customElFilter] - Additional filter for element
+ * @param {Boolean} [skipAttrFilters] - Set to true, if expect not filtered list of resources
+ *
+ * @returns {Array} Returns array with HTML of Style containers OR undefined
+ */
+var filterResourceElements = function($, $elements, customElFilter, skipAttrFilters){
+    var filteredArr = [];
+    var defaultFilter = function () {
+        return true;
+    };
+    var _customElFilter = typeof customElFilter === 'function' ? customElFilter : defaultFilter;
+
+    // Checking some exceptions
+    var checkDataSet = function($el){
+        var elementAttrs = $el.attributes || $el.attribs;
+
+        if (skipAttrFilters) {
+            return true;
+        } else {
+            return !(
+                elementAttrs['data-nonclarify'] ||
+                elementAttrs['data-requiremodule'] ||
+                elementAttrs['data-source'] === 'core' ||
+                elementAttrs['data-source'] === 'plugin'
+            );
+        }
+    };
+
+    if ($elements && $elements.length > 0) {
+        $elements.each(function(){
+           var el = this;
+
+            if (_customElFilter(el) && checkDataSet(el)) {
+                // If not need to filter out, then push to output
+                filteredArr.push($.html($(el)));
+            }
+        });
+
+        return filteredArr.length > 0 ? filteredArr : undefined;
+    } else {
+        return undefined;
+    }
+};
+
+/**
+ * Parse CSS links from Spec
+ *
+ * @param {Object} $ - cheerio root object
+ * @param {Object} scope to search - body or head, defaults to root
+ * @param {Boolean} [skipAttrFilters] - Set to true, if expect not filtered list of resources
+ *
+ * @returns {Array} Returns array with HTML of CSS links OR undefined
+ */
+var getCssLinksHTML = module.exports.getCssLinksHTML = function($, scope, skipAttrFilters) {
+    var $scope = scope ? $(scope) : $.root();
+    var $links = $scope.find('link');
+
+    var customFilter = function(el){
+        var elementAttrs = el.attributes || el.attribs;
+
+        return elementAttrs.rel === 'stylesheet';
+    };
+
+    return filterResourceElements($, $links, customFilter, skipAttrFilters);
+};
+
+
+/**
+ * Parse Scripts from Spec
+ *
+ * @param {Object} $ - cheerio root object
+ * @param {Object} scope to search - body or head, defaults to root
+ * @param {Boolean} [skipAttrFilters] - Set to true, if expect not filtered list of resources
+ *
+ * @returns {Array} Returns array with HTML of Scripts OR undefined
+ */
+var getScriptsHTML = module.exports.getScriptsHTML = function($, scope, skipAttrFilters) {
+    var $scope = scope ? $(scope) : $.root();
+    var $scripts = $scope.find('script');
+
+    return filterResourceElements($, $scripts, null, skipAttrFilters);
+};
+
+/**
+ * Parse Style containers from Spec
+ *
+ * @param {Object} $ - cheerio root object
+ * @param {Object} scope to search - body or head, defaults to root
+ * @param {Boolean} [skipAttrFilters] - Set to true, if expect not filtered list of resources
+ *
+ * @returns {Array} Returns array with HTML of Style containers OR undefined
+ */
+var getStyleContainersHTML = module.exports.getStyleContainersHTML = function($, scope, skipAttrFilters){
+    var $scope = scope ? $(scope) : $.root();
+    var $styles = $scope.find('style');
+
+    return filterResourceElements($, $styles, null, skipAttrFilters);
+};
+
+
+/**
+ * Getter for assets resources of the Spec
+ *
+ * @param {Object} $ - cheerio root object
+ * @param {String} $scope - cheerio DOM object
+ *
+ * @returns {Object} Returns object with HTML lists containing Spec assets resources OR undefined
+ */
+var getSpecResources = module.exports.getSpecResources = function ($, scope) {
+    var output = {};
+
+    output.cssLinks = getCssLinksHTML($, scope) || [];
+    output.scripts = getScriptsHTML($, scope) || [];
+    output.cssStyles = getStyleContainersHTML($, scope) || [];
+
+    return (output.cssLinks || output.scripts || output.cssStyles) ? output : undefined;
+};
+
 /**
  * Parse Spec page contents into HTML tree
  *
- * @param {String} specHTML - Full Spec page html (from request)
+ * @param {String} specHTML - Full Spec page html, including head section (from request)
  *
  * @returns {Object} Return parsed Spec page object
  */
 module.exports.process = function (specHTML) {
+    var $ = cheerio.load(specHTML, {decodeEntities: false});
     var output = {
-        headResources: {},
-        bodyResources: {},
+        headResources: getSpecResources($, 'head'),
+        bodyResources: getSpecResources($, 'body'),
         contents: []
     };
 
-    var $ = cheerio.load(specHTML, {decodeEntities: false});
-
     var parseBlock = function (elements, blockID, blockDefaultID, nesting) {
         nesting = nesting || 0;
 
@@ -83,4 +205,4 @@ module.exports.process = function (specHTML) {
  */
 module.exports.getBySection = function (specObject, sections) {
     return ParseData.traverseSections(specObject, sections);
-};
\ No newline at end of file
+};
diff --git a/test/unit/partials/markdown-ejs-nested.md b/test/partials/markdown-ejs-nested.md
similarity index 100%
rename from test/unit/partials/markdown-ejs-nested.md
rename to test/partials/markdown-ejs-nested.md
diff --git a/test/unit/partials/markdown-ejs.md b/test/partials/markdown-ejs.md
similarity index 100%
rename from test/unit/partials/markdown-ejs.md
rename to test/partials/markdown-ejs.md
diff --git a/test/unit/partials/markdown.md b/test/partials/markdown.md
similarity index 100%
rename from test/unit/partials/markdown.md
rename to test/partials/markdown.md
diff --git a/test/unit/partials/mask-one.html b/test/partials/mask-one.html
similarity index 100%
rename from test/unit/partials/mask-one.html
rename to test/partials/mask-one.html
diff --git a/test/unit/partials/mask-two.html b/test/partials/mask-two.html
similarity index 100%
rename from test/unit/partials/mask-two.html
rename to test/partials/mask-two.html
diff --git a/test/unit/partials/specs/complex/index.src.html b/test/partials/specs/complex/index.src.html
similarity index 100%
rename from test/unit/partials/specs/complex/index.src.html
rename to test/partials/specs/complex/index.src.html
diff --git a/test/partials/specs/complex/info.json b/test/partials/specs/complex/info.json
new file mode 100644
index 0000000..2fbaeb3
--- /dev/null
+++ b/test/partials/specs/complex/info.json
@@ -0,0 +1,5 @@
+{
+    "title": "Nested Spec",
+    "template": "doc",
+    "tag": ["hidden"]
+}
diff --git a/test/unit/partials/specs/few-examples/index.src.html b/test/partials/specs/few-examples/index.src.html
similarity index 100%
rename from test/unit/partials/specs/few-examples/index.src.html
rename to test/partials/specs/few-examples/index.src.html
diff --git a/test/partials/specs/few-examples/info.json b/test/partials/specs/few-examples/info.json
new file mode 100644
index 0000000..6830b86
--- /dev/null
+++ b/test/partials/specs/few-examples/info.json
@@ -0,0 +1,5 @@
+{
+    "title": "Few-examples Spec",
+    "template": "doc",
+    "tag": ["hidden"]
+}
diff --git a/test/unit/partials/specs/simple/index.src.html b/test/partials/specs/many-assets/index.src.html
similarity index 100%
rename from test/unit/partials/specs/simple/index.src.html
rename to test/partials/specs/many-assets/index.src.html
diff --git a/test/partials/specs/many-assets/info.json b/test/partials/specs/many-assets/info.json
new file mode 100644
index 0000000..237f4a3
--- /dev/null
+++ b/test/partials/specs/many-assets/info.json
@@ -0,0 +1,5 @@
+{
+    "title": "Many Assets",
+    "template": "$(context)/template.ejs",
+    "tag": ["hidden"]
+}
diff --git a/test/partials/specs/many-assets/template.ejs b/test/partials/specs/many-assets/template.ejs
new file mode 100644
index 0000000..fc3a81c
--- /dev/null
+++ b/test/partials/specs/many-assets/template.ejs
@@ -0,0 +1,48 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <title><%= info.title %></title>
+    <meta name="author" content="<%= info.author %>">
+    <% if (info.keywords !== '') {%>
+    <meta name="keywords" content="<%= info.keywords %>">
+    <% } %>
+
+    <!-- Your master-app styles -->
+        <link href="/docs/data/bootstrap-head.css" rel="stylesheet">
+        <script src="script-in-head.js"></script>
+        <style>.head{color:red}</style>
+
+        <link href="/docs/data/bootstrap-head.css" rel="stylesheet" data-nonclarify="true">
+        <script src="script-in-head.js" data-nonclarify="true"></script>
+        <style  data-nonclarify="true">.head{color:red}</style>
+    <!-- /Your master-app styles -->
+
+    <!-- Source default resources -->
+        <link href="/source/assets/css/source.css" rel="stylesheet" data-source="core">
+        <script src="/source/assets/js/require.bundle.js" data-source="core"></script>
+        <script src="/source/assets/js/enter-the-source.js" data-source="core"></script>
+    <!-- /Source default resources -->
+
+    <%- head %>
+</head>
+<body class="source_page __spec">
+    <link href="/docs/data/bootstrap-body.css" rel="stylesheet">
+    <div class="source_container">
+        <%- header %>
+        <div class="source_main source_col-main" role="main">
+            <h1><%= info.title %></h1>
+            <div class="source_header-meta">v.<%- engineVersion %></div>
+            <%- content %>
+        </div>
+        <%- footer %>
+        <style>.body{color:red}</style>
+    </div>
+    <script src="script-in-body.js"></script>
+
+    <link href="/docs/data/bootstrap-head.css" rel="stylesheet" data-nonclarify="true">
+    <script src="script-in-head.js" data-nonclarify="true"></script>
+    <style data-nonclarify="true">.body{color:red}</style>
+</body>
+</html>
\ No newline at end of file
diff --git a/test/unit/partials/specs/nested/index.src.html b/test/partials/specs/nested/index.src.html
similarity index 100%
rename from test/unit/partials/specs/nested/index.src.html
rename to test/partials/specs/nested/index.src.html
diff --git a/test/partials/specs/nested/info.json b/test/partials/specs/nested/info.json
new file mode 100644
index 0000000..2fbaeb3
--- /dev/null
+++ b/test/partials/specs/nested/info.json
@@ -0,0 +1,5 @@
+{
+    "title": "Nested Spec",
+    "template": "doc",
+    "tag": ["hidden"]
+}
diff --git a/test/partials/specs/simple/index.src.html b/test/partials/specs/simple/index.src.html
new file mode 100644
index 0000000..45ca756
--- /dev/null
+++ b/test/partials/specs/simple/index.src.html
@@ -0,0 +1,5 @@
+<div class="source_section">
+    <h2>Section header</h2>
+
+    <div class="source_example">HTML example</div>
+</div>
\ No newline at end of file
diff --git a/test/partials/specs/simple/info.json b/test/partials/specs/simple/info.json
new file mode 100644
index 0000000..f2a5c8b
--- /dev/null
+++ b/test/partials/specs/simple/info.json
@@ -0,0 +1,5 @@
+{
+    "title": "Simple Spec",
+    "template": "doc",
+    "tag": ["hidden"]
+}
diff --git a/test/unit/partials/three.html b/test/partials/three.html
similarity index 100%
rename from test/unit/partials/three.html
rename to test/partials/three.html
diff --git a/test/unit/ejsHelpersSpec.js b/test/unit/ejsHelpersSpec.js
index 803cd29..7224f67 100644
--- a/test/unit/ejsHelpersSpec.js
+++ b/test/unit/ejsHelpersSpec.js
@@ -9,7 +9,7 @@ global.opts = loadOptions(path.resolve(pathToMasterApp));
 
 describe('includeMD', function () {
     it('should properly include Markdown file', function (done) {
-        var result = ejs.render('<%- includeMD("partials/markdown") %>', {
+        var result = ejs.render('<%- includeMD("../partials/markdown") %>', {
             filename: __filename
         });
 
@@ -19,7 +19,7 @@ describe('includeMD', function () {
     });
 
     it('should properly include Markdown file with EJS includes', function (done) {
-        var result = ejs.render('<%- includeMD("partials/markdown-ejs") %>', {
+        var result = ejs.render('<%- includeMD("../partials/markdown-ejs") %>', {
           world: 'world'
         }, {
             filename: __filename
@@ -31,7 +31,7 @@ describe('includeMD', function () {
     });
 
     it('should properly include Markdown with nested include', function (done) {
-        var result = ejs.render('<%- includeMD("partials/markdown-ejs-nested") %>', {
+        var result = ejs.render('<%- includeMD("../partials/markdown-ejs-nested") %>', {
           world: 'world'
         }, {
             filename: __filename
@@ -45,7 +45,7 @@ describe('includeMD', function () {
 
 describe('includeFiles', function () {
     it('should include one file by glob', function (done) {
-        var result = ejs.render('<%- includeFiles("partials/mask-one.html") %>', {
+        var result = ejs.render('<%- includeFiles("../partials/mask-one.html") %>', {
             filename: __filename
         });
 
@@ -55,7 +55,7 @@ describe('includeFiles', function () {
     });
 
     it('should include multiple files by glob', function (done) {
-        var result = ejs.render('<%- includeFiles("partials/mask-*.html") %>', {
+        var result = ejs.render('<%- includeFiles("../partials/mask-*.html") %>', {
             filename: __filename
         });
 
diff --git a/test/unit/lib/specPageParserSpec.js b/test/unit/lib/specPageParserSpec.js
index c263b96..daa144e 100644
--- a/test/unit/lib/specPageParserSpec.js
+++ b/test/unit/lib/specPageParserSpec.js
@@ -1,4 +1,5 @@
 var path = require('path');
+var cheerio = require('cheerio');
 var expect = require('chai').expect;
 
 var pathToMasterApp = path.resolve('./');
@@ -8,13 +9,240 @@ global.opts = loadOptions(path.resolve(pathToMasterApp));
 var specsParser = require(path.join(pathToMasterApp, 'core/lib/specPageParser'));
 var request = require(path.join(pathToMasterApp, 'core/lib/request'));
 
+describe('specsParser.getCssLinksHTML', function () {
+    it('it should parse all css links html', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getCssLinksHTML($);
+
+                expect(result).to.be.an('array');
+                expect(result[0]).to.equal('<link href="/docs/data/bootstrap-head.css" rel="stylesheet">');
+                expect(result[1]).to.equal('<link href="/docs/data/bootstrap-body.css" rel="stylesheet">');
+
+                done();
+            }
+        });
+    });
+    it('it should parse css links only from head', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getCssLinksHTML($, 'head');
+
+                expect(result).to.be.an('array');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal('<link href="/docs/data/bootstrap-head.css" rel="stylesheet">');
+
+                done();
+            }
+        });
+    });
+    it('it should parse css links only from head', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getCssLinksHTML($, 'body');
+
+                expect(result).to.be.an('array');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal('<link href="/docs/data/bootstrap-body.css" rel="stylesheet">');
+
+                done();
+            }
+        });
+    });
+});
+
+describe('specsParser.getScriptsHTML', function () {
+    it('it should parse all scripts in html', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getScriptsHTML($);
+
+                expect(result).to.be.an('array');
+                expect(result[0]).to.equal('<script src="script-in-head.js"></script>');
+                expect(result[1]).to.equal('<script src="script-in-body.js"></script>');
+
+                done();
+            }
+        });
+    });
+    it('it should parse scriptsonly from head', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getScriptsHTML($, 'head');
+
+                expect(result).to.be.an('array');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal('<script src="script-in-head.js"></script>');
+
+                done();
+            }
+        });
+    });
+    it('it should parse scripts only from head', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getScriptsHTML($, 'body');
+
+                expect(result).to.be.an('array');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal('<script src="script-in-body.js"></script>');
+
+                done();
+            }
+        });
+    });
+});
+
+describe('specsParser.getStyleContainersHTML', function () {
+    it('it should parse all style containers html', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getStyleContainersHTML($);
+
+                expect(result).to.be.an('array');
+                expect(result[0]).to.equal('<style>.head{color:red}</style>');
+                expect(result[1]).to.equal('<style>.body{color:red}</style>');
+
+                done();
+            }
+        });
+    });
+    it('it should parse styles only from head', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getStyleContainersHTML($, 'head');
+
+                expect(result).to.be.an('array');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal('<style>.head{color:red}</style>');
+
+                done();
+            }
+        });
+    });
+    it('it should parse styles only from head', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getStyleContainersHTML($, 'body');
+
+                expect(result).to.be.an('array');
+                expect(result.length).to.equal(1);
+                expect(result[0]).to.equal('<style>.body{color:red}</style>');
+
+                done();
+            }
+        });
+    });
+});
+
+describe('specsParser.getSpecResources', function () {
+    it('it should parse all resources', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getSpecResources($);
+
+                expect(result).to.be.an('object');
+                expect(result.cssLinks).to.be.an('array');
+                expect(result.scripts).to.be.an('array');
+                expect(result.cssStyles).to.be.an('array');
+
+                expect(result.cssLinks.length).to.equal(2);
+                expect(result.scripts.length).to.equal(2);
+                expect(result.cssStyles.length).to.equal(2);
+
+                done();
+            }
+        });
+    });
+    it('it should parse all resources from head', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getSpecResources($, 'head');
+
+                expect(result).to.be.an('object');
+                expect(result.cssLinks).to.be.an('array');
+                expect(result.scripts).to.be.an('array');
+                expect(result.cssStyles).to.be.an('array');
+
+                expect(result.cssLinks.length).to.equal(1);
+                expect(result.scripts.length).to.equal(1);
+                expect(result.cssStyles.length).to.equal(1);
+
+                done();
+            }
+        });
+    });
+    it('it should parse all resources from body', function (done) {
+        request({
+            path: '/test/partials/specs/many-assets/',
+            internal: true,
+            callback: function (response) {
+                var $ = cheerio.load(response, {decodeEntities: false});
+                var result = specsParser.getSpecResources($, 'head');
+
+                expect(result).to.be.an('object');
+                expect(result.cssLinks).to.be.an('array');
+                expect(result.scripts).to.be.an('array');
+                expect(result.cssStyles).to.be.an('array');
+
+                expect(result.cssLinks.length).to.equal(1);
+                expect(result.scripts.length).to.equal(1);
+                expect(result.cssStyles.length).to.equal(1);
+
+                done();
+            }
+        });
+    });
+});
+
 describe('specsParser.process', function () {
     it('it should return proper object structure', function (done) {
         var result = specsParser.process('');
 
         expect(result).to.be.an('object');
-        expect(result.headResources).to.be.an('object');
-        expect(result.bodyResources).to.be.an('object');
+            expect(result.headResources).to.be.an('object');
+                expect(result.headResources.cssLinks).to.be.an('array');
+                expect(result.headResources.cssStyles).to.be.an('array');
+                expect(result.headResources.scripts).to.be.an('array');
+
+            expect(result.bodyResources).to.be.an('object');
+                expect(result.bodyResources.cssLinks).to.be.an('array');
+                expect(result.bodyResources.cssStyles).to.be.an('array');
+                expect(result.bodyResources.scripts).to.be.an('array');
+
         expect(result.contents).to.be.an('array');
 
         done();
@@ -22,24 +250,20 @@ describe('specsParser.process', function () {
 
     it('it should return parsed spec object', function (done) {
         request({
-            path: '/test/unit/partials/specs/simple/',
+            path: '/test/partials/specs/simple/',
             internal: true,
             callback: function (response) {
                 var result = specsParser.process(response);
 
-                expect(result).to.deep.equal({
-                    headResources: {},
-                    bodyResources: {},
-                    contents: [{
-                        nested: [],
-                        html: [
-                            'HTML example'
-                        ],
-                        id: '1',
-                        visualID: '1',
-                        header: 'Section header'
-                    }]
-                });
+                expect(result.contents).to.deep.equal([{
+                    nested: [],
+                    html: [
+                        'HTML example'
+                    ],
+                    id: '1',
+                    visualID: '1',
+                    header: 'Section header'
+                }]);
 
                 done();
             }
@@ -48,25 +272,21 @@ describe('specsParser.process', function () {
 
     it('it should return parsed spec object with few examples', function (done) {
         request({
-            path: '/test/unit/partials/specs/few-examples/',
+            path: '/test/partials/specs/few-examples/',
             internal: true,
             callback: function (response) {
                 var result = specsParser.process(response);
 
-                expect(result).to.deep.equal({
-                    headResources: {},
-                    bodyResources: {},
-                    contents: [{
-                        nested: [],
-                        html: [
-                            'HTML example 1',
-                            'HTML example 2'
-                        ],
-                        id: '1',
-                        visualID: '1',
-                        header: 'Section header'
-                    }]
-                });
+                expect(result.contents).to.deep.equal([{
+                    nested: [],
+                    html: [
+                        'HTML example 1',
+                        'HTML example 2'
+                    ],
+                    id: '1',
+                    visualID: '1',
+                    header: 'Section header'
+                }]);
 
                 done();
             }
@@ -75,7 +295,7 @@ describe('specsParser.process', function () {
 
     it('it should properly parse nested IDs', function (done) {
         request({
-            path: '/test/unit/partials/specs/nested/',
+            path: '/test/partials/specs/nested/',
             internal: true,
             callback: function (response) {
                 var result = specsParser.process(response);
@@ -97,7 +317,7 @@ describe('specsParser.process', function () {
 
     it('it should properly parse multiple custom ID\'s', function (done) {
         request({
-            path: '/test/unit/partials/specs/complex/',
+            path: '/test/partials/specs/complex/',
             internal: true,
             callback: function (response) {
                 var result = specsParser.process(response);
@@ -115,7 +335,7 @@ describe('specsParser.process', function () {
 
     it('it should properly pass multiple HTML examples', function (done) {
         request({
-            path: '/test/unit/partials/specs/complex/',
+            path: '/test/partials/specs/complex/',
             internal: true,
             callback: function (response) {
                 var result = specsParser.process(response);
@@ -134,14 +354,24 @@ describe('specsParser.process', function () {
 
     it('it should return full nested spec object', function (done) {
         request({
-            path: '/test/unit/partials/specs/nested/',
+            path: '/test/partials/specs/nested/',
             internal: true,
             callback: function (response) {
                 var result = specsParser.process(response);
 
                 expect(result).to.deep.equal({
-                    headResources: {},
-                    bodyResources: {},
+                    "headResources": {
+                        "cssLinks": [
+                            "<link href=\"/docs/data/bootstrap.css\" rel=\"stylesheet\">"
+                        ],
+                        "cssStyles": [],
+                        "scripts": []
+                    },
+                    bodyResources: {
+                        "cssLinks": [],
+                        "cssStyles": [],
+                        "scripts": []
+                    },
                     contents: [{
                         html: [
                             'HTML example 1'
@@ -187,13 +417,25 @@ describe('specsParser.process', function () {
 describe('specsParser.getBySection', function () {
     it('it should properly get by single section', function (done) {
         request({
-            path: '/test/unit/partials/specs/nested/',
+            path: '/test/partials/specs/nested/',
             internal: true,
             callback: function (response) {
                 var result = specsParser.process(response);
                 var assertOption = function (sections) {
                     var endResult = specsParser.getBySection(result, sections);
 
+                    expect(result).to.be.an('object');
+
+                    expect(result.headResources).to.be.an('object');
+                    expect(result.headResources.cssLinks).to.be.an('array');
+                    expect(result.headResources.cssStyles).to.be.an('array');
+                    expect(result.headResources.scripts).to.be.an('array');
+
+                    expect(result.bodyResources).to.be.an('object');
+                    expect(result.bodyResources.cssLinks).to.be.an('array');
+                    expect(result.bodyResources.cssStyles).to.be.an('array');
+                    expect(result.bodyResources.scripts).to.be.an('array');
+
                     expect(endResult.contents.length).to.equal(1);
                     expect(endResult.contents[0].id).to.equal(sections[0]);
                 };
diff --git a/test/unit/middlewares/clarifySpec.js b/test/unit/middlewares/clarifySpec.js
index 92e02cb..e603919 100644
--- a/test/unit/middlewares/clarifySpec.js
+++ b/test/unit/middlewares/clarifySpec.js
@@ -92,7 +92,7 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                 });
         });
 
-        xit('should have injected resources', function (done) {
+        it('should have injected resources', function (done) {
             request(url)
                 .get('&sections=1.1')
                 .expect(200)
diff --git a/test/unit/partials/specs/complex/info.json b/test/unit/partials/specs/complex/info.json
deleted file mode 100644
index 071dd19..0000000
--- a/test/unit/partials/specs/complex/info.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "title": "Nested Spec",
-    "template": "doc"
-}
diff --git a/test/unit/partials/specs/few-examples/info.json b/test/unit/partials/specs/few-examples/info.json
deleted file mode 100644
index 5bf65ff..0000000
--- a/test/unit/partials/specs/few-examples/info.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "title": "Few-examples Spec",
-    "template": "doc"
-}
diff --git a/test/unit/partials/specs/nested/info.json b/test/unit/partials/specs/nested/info.json
deleted file mode 100644
index 071dd19..0000000
--- a/test/unit/partials/specs/nested/info.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "title": "Nested Spec",
-    "template": "doc"
-}
diff --git a/test/unit/partials/specs/simple/info.json b/test/unit/partials/specs/simple/info.json
deleted file mode 100644
index 695a230..0000000
--- a/test/unit/partials/specs/simple/info.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "title": "Simple Spec",
-    "template": "doc"
-}

From 416025ba4263f3e79d7e690cef2bac15f98d5cd4 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 18:12:29 +0200
Subject: [PATCH 10/47] fix linting issue

---
 core/middlewares/wrap.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/middlewares/wrap.js b/core/middlewares/wrap.js
index 99d66e5..a2d6cf5 100644
--- a/core/middlewares/wrap.js
+++ b/core/middlewares/wrap.js
@@ -46,7 +46,7 @@ exports.process = function (req, res, next) {
                 var pathToCheck = path.join(global.app.get('user'), includedDir) + '/';
 
                 if (context.indexOf(pathToCheck) === 0) {
-                    context = context.replace(pathToCheck, path.join(global.pathToApp, includedDir) + '/')
+                    context = context.replace(pathToCheck, path.join(global.pathToApp, includedDir) + '/');
                 }
             });
         }

From 899caf77d3215cf96544072b9878e0e9a1f67304 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 18:51:46 +0200
Subject: [PATCH 11/47] fix: return missing list of ID's

---
 core/middlewares/clarify.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/middlewares/clarify.js b/core/middlewares/clarify.js
index 308fa81..4838d0c 100644
--- a/core/middlewares/clarify.js
+++ b/core/middlewares/clarify.js
@@ -35,7 +35,7 @@ var getTpl = function(tpl) {
     var pathToTemplate = path.join(pathToApp, 'core/views/clarify', templateName + '.ejs');
     var userPathToTemplate = path.join(global.app.get('user'), 'core/views/clarify', templateName + '.ejs');
 
-    // First we check user tempalte, then core
+    // First we check user tempalate, then core
     fs.readFile(userPathToTemplate, 'utf-8', function(err, data){
         if (err) {
 
@@ -124,7 +124,7 @@ var parseSpec = function(sections, pathToSpec) {
             var allContents = specsParser.process(specHTML);
 
             if (sections) {
-                output = specsParser.getBySection(allContents, sections);
+                output = specsParser.getBySection(_.merge({}, allContents), sections);
             } else {
                 output = allContents;
             }

From cee60d1fcc7b0b8cb9c8c8ddb16ba5f1353510f8 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 19:03:31 +0200
Subject: [PATCH 12/47] #99: removing last bits of jsdom

---
 docs/base/readme.md                  |  24 ----
 docs/clarify/readme.md               |   2 +-
 package.json                         |   3 +-
 test/unit/middlewares/clarifySpec.js | 171 ++++++++-------------------
 4 files changed, 54 insertions(+), 146 deletions(-)

diff --git a/docs/base/readme.md b/docs/base/readme.md
index b02b387..1885108 100644
--- a/docs/base/readme.md
+++ b/docs/base/readme.md
@@ -31,30 +31,6 @@ npm install sourcejs --save
 npm start
 ```
 
-Starting from 0.6.0 we're planning to change official install path to one with NPM packages. Yeoman generator will be also replaced by [sourcejs-cli](https://github.com/sourcejs/sourcejs-cli) with commands like `run`, `install` (plugin), `init`, `create`.
-
-### Installing On Windows
-
-If you're running Windows and have some issues with [JSDom](https://github.com/tmpvar/jsdom) dependencies compilation, please check this [topic](https://github.com/sourcejs/Source/issues/23).
-
-Alternatively with 0.5.6 we prepared a special build without JSDom, until it's full removal from core at 0.6.0.
-
-```html
-npm install -g yo generator-sourcejs
-mkdir sourcejs && cd sourcejs
-yo sourcejs --branch 0.5.6-no-jsdom
-```
-
-Please note that Clarify feature is not available in `no-jsdom` version. If you had generator installed before, run `npm update -g generator-sourcejs` (v.0.4.2+ required).
-
-And installing same build from NPM:
-
-```html
-git clone https://github.com/sourcejs/init.git -b npm my-sourcejs && cd my-sourcejs
-npm install sourcejs@0.5.6-no-jsdom --save
-npm start
-```
-
 ## Commands
 
 ### Run
diff --git a/docs/clarify/readme.md b/docs/clarify/readme.md
index 544c68e..0346c87 100644
--- a/docs/clarify/readme.md
+++ b/docs/clarify/readme.md
@@ -8,7 +8,7 @@ Clarify is an [expressJS](http://expressjs.com/) middleware built into SourceJS
 http://localhost:8080/docs/spec/?clarify=true&sections=1.1
 ```
 
-When enabled, Clarify uses [JSdom](https://github.com/tmpvar/jsdom) for getting specified sections and wrap them in pre-defined or user templates. If you're using SourceJS HTML API, Clarify can be configured to take data directly from API storage as well.
+When enabled, Clarify uses [Cheerio](https://github.com/cheeriojs/cheerio) for getting specified sections and wrap them in pre-defined or user templates. If you're using SourceJS HTML API, Clarify can be configured to take data directly from API storage as well.
 
 Clarify page is enhanced with helper panel, where you can chose any option available:
 
diff --git a/package.json b/package.json
index f25779d..7dbff8a 100644
--- a/package.json
+++ b/package.json
@@ -58,8 +58,7 @@
     "shortid": "^2.2.2",
     "time-grunt": "~0.2.10",
     "tinyforever": "0.0.3",
-    "universal-analytics": "^0.3.9",
-    "jsdom": "^1.2.1"
+    "universal-analytics": "^0.3.9"
   },
   "scripts": {
     "postinstall": "node ./core/postInstall && grunt update && node ./core/trackInstall",
diff --git a/test/unit/middlewares/clarifySpec.js b/test/unit/middlewares/clarifySpec.js
index e603919..e1fc221 100644
--- a/test/unit/middlewares/clarifySpec.js
+++ b/test/unit/middlewares/clarifySpec.js
@@ -2,21 +2,19 @@ var fs = require('fs');
 var path = require('path');
 
 var pathToMasterApp = path.resolve('./');
-var jq = fs.readFileSync(path.join(pathToMasterApp,'assets/js/lib/jquery-2.1.4.min.js'), "utf-8");
 
 var should = require('should');
 var assert = require('assert');
 var request = require('supertest');
 
-// TODO: rewrite tests with cheerio
-var jsdom = require('jsdom');
+var cheerio = require('cheerio');
 
 var loadOptions = require(path.join(pathToMasterApp, 'core/loadOptions'));
 global.opts = loadOptions(path.resolve(pathToMasterApp));
 
 
-describe('Clarify test /docs/spec?clarify=true', function () {
-    describe('GET from JSDOM /docs/spec?clarify=true...', function () {
+describe.only('Clarify test /docs/spec?clarify=true', function () {
+    describe('GET /docs/spec?clarify=true...', function () {
         var url = 'http://localhost:8080/docs/spec/?clarify=true';
         it('should return nothing (&sections=77)', function (done) {
             request(url)
@@ -27,18 +25,11 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                         throw err;
                     }
 
-                    jsdom.env({
-                        html: res.text,
-                        src: [jq],
-                        done: function (errors, window) {
-                            var $ = window.$;
-                            var examples = $('.source_example');
+                    var $ = cheerio.load(res.text, {decodeEntities: false});
+                    var examples = $('.source_example');
 
-                            examples.length.should.equal(0);
-
-                            done();
-                        }
-                    });
+                    examples.length.should.equal(0);
+                    done();
                 });
         });
 
@@ -51,20 +42,14 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                         throw err;
                     }
 
-                    jsdom.env({
-                        html: res.text,
-                        src: [jq],
-                        done: function (errors, window) {
-                            var $ = window.$;
-                            var sectionHeaders = $('.source_section_h');
-                            var examples = $('.source_example');
+                    var $ = cheerio.load(res.text, {decodeEntities: false});
+                    var sectionHeaders = $('.source_section_h');
+                    var examples = $('.source_example');
 
-                            sectionHeaders.length.should.be.greaterThan(1);
-                            examples.length.should.be.greaterThan(0);
+                    sectionHeaders.length.should.be.greaterThan(1);
+                    examples.length.should.be.greaterThan(0);
 
-                            done();
-                        }
-                    });
+                    done();
                 });
         });
 
@@ -77,18 +62,10 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                         throw err;
                     }
 
-                    jsdom.env({
-                        html: res.text,
-                        src: [jq],
-                        done: function (errors, window) {
-                            var $ = window.$;
-                            var examples = $('.source_example');
-
-                            examples.length.should.equal(1);
-
-                            done();
-                        }
-                    });
+                    var $ = cheerio.load(res.text, {decodeEntities: false});
+                    var examples = $('.source_example');
+                    examples.length.should.equal(1);
+                    done();
                 });
         });
 
@@ -101,19 +78,12 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                         throw err;
                     }
 
-                    jsdom.env({
-                        html: res.text,
-                        src: [jq],
-                        done: function (errors, window) {
-                            var $ = window.$;
-
-                            $('[href*="bootstrap.css"]').length.should.equal(1);
-                            $('body > style').length.should.be.greaterThan(0);
-                            $('body > script').length.should.be.greaterThan(0);
+                    var $ = cheerio.load(res.text, {decodeEntities: false});
+                    $('[href*="bootstrap.css"]').length.should.equal(1);
+                    $('body > style').length.should.be.greaterThan(0);
+                    $('body > script').length.should.be.greaterThan(0);
 
-                            done();
-                        }
-                    });
+                    done();
                 });
         });
 
@@ -126,19 +96,13 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                         throw err;
                     }
 
-                    jsdom.env({
-                        html: res.text,
-                        src: [jq],
-                        done: function (errors, window) {
-                            var $ = window.$;
-                            var examples = $('.source_example');
+                    var $ = cheerio.load(res.text, {decodeEntities: false});
 
-                            $('.__clear').length.should.equal(1);
-                            examples.length.should.equal(0);
+                    var examples = $('.source_example');
 
-                            done();
-                        }
-                    });
+                    $('.__clear').length.should.equal(1);
+                    examples.length.should.equal(0);
+                    done();
                 });
         });
     });
@@ -156,18 +120,10 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                         throw err;
                     }
 
-                    jsdom.env({
-                        html: res.text,
-                        src: [jq],
-                        done: function (errors, window) {
-                            var $ = window.$;
-                            var examples = $('.source_example');
-
-                            examples.length.should.equal(0);
-
-                            done();
-                        }
-                    });
+                    var $ = cheerio.load(res.text, {decodeEntities: false});
+                    var examples = $('.source_example');
+                    examples.length.should.equal(0);
+                    done();
                 });
         });
 
@@ -181,20 +137,14 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                             throw err;
                         }
 
-                        jsdom.env({
-                            html: res.text,
-                            src: [jq],
-                            done: function (errors, window) {
-                                var $ = window.$;
-                                var sectionHeaders = $('.source_section_h');
-                                var examples = $('.source_example');
+                        var $ = cheerio.load(res.text, {decodeEntities: false});
+                        var sectionHeaders = $('.source_section_h');
+                        var examples = $('.source_example');
 
-                                sectionHeaders.length.should.be.greaterThan(1);
-                                examples.length.should.be.greaterThan(0);
+                        sectionHeaders.length.should.be.greaterThan(1);
+                        examples.length.should.be.greaterThan(0);
 
-                                done();
-                            }
-                        });
+                        done();
                     });
             });
 
@@ -207,18 +157,12 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                             throw err;
                         }
 
-                        jsdom.env({
-                            html: res.text,
-                            src: [jq],
-                            done: function (errors, window) {
-                                var $ = window.$;
-                                var examples = $('.source_example');
+                        var $ = cheerio.load(res.text, {decodeEntities: false});
+                        var examples = $('.source_example');
 
-                                examples.length.should.equal(1);
+                        examples.length.should.equal(1);
 
-                                done();
-                            }
-                        });
+                        done();
                     });
             });
 
@@ -231,19 +175,13 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                             throw err;
                         }
 
-                        jsdom.env({
-                            html: res.text,
-                            src: [jq],
-                            done: function (errors, window) {
-                                var $ = window.$;
+                        var $ = cheerio.load(res.text, {decodeEntities: false});
 
-                                $('[href*="bootstrap.css"]').length.should.equal(1);
-                                $('body > style').length.should.be.greaterThan(0);
-                                $('body > script').length.should.be.greaterThan(0);
+                        $('[href*="bootstrap.css"]').length.should.equal(1);
+                        $('body > style').length.should.be.greaterThan(0);
+                        $('body > script').length.should.be.greaterThan(0);
 
-                                done();
-                            }
-                        });
+                        done();
                     });
             });
 
@@ -256,19 +194,14 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                             throw err;
                         }
 
-                        jsdom.env({
-                            html: res.text,
-                            src: [jq],
-                            done: function (errors, window) {
-                                var $ = window.$;
-                                var examples = $('.source_example');
+                        var $ = cheerio.load(res.text, {decodeEntities: false});
+
+                        var examples = $('.source_example');
 
-                                $('.__clear').length.should.equal(1);
-                                examples.length.should.equal(0);
+                        $('.__clear').length.should.equal(1);
+                        examples.length.should.equal(0);
 
-                                done();
-                            }
-                        });
+                        done();
                     });
             });
         }

From 0aad6900b9479405f70d3d6c2246ad275fb1cd11 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 19:41:35 +0200
Subject: [PATCH 13/47] improve clarify tests

---
 Gruntfile.js                         |  2 +-
 test/functional/clarify.js           | 44 ++++++++++++++++++++++++++++
 test/unit/middlewares/clarifySpec.js | 32 +++++++++++++++++---
 3 files changed, 73 insertions(+), 5 deletions(-)
 create mode 100644 test/functional/clarify.js

diff --git a/Gruntfile.js b/Gruntfile.js
index 5fca981..8fb2f0d 100755
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -340,7 +340,7 @@ module.exports = function(grunt) {
 
     // Test task. Execute with running app
     grunt.registerTask('test-func', 'Run ALL functional tests or specified by second param', function () {
-        // if custom mask set - `grunt test --spec=test/unit/**/*.js`
+        // if custom mask set - `grunt test --spec=test/functional/**/*.js`
         var spec = grunt.option('spec');
         if (spec) {
             grunt.config.set('casperjs.files', [spec]);
diff --git a/test/functional/clarify.js b/test/functional/clarify.js
new file mode 100644
index 0000000..738e1dd
--- /dev/null
+++ b/test/functional/clarify.js
@@ -0,0 +1,44 @@
+var appPort = casper.cli.get('app-port') || 8080;
+var url = 'http://127.0.0.1:' + appPort + '/docs/spec/';
+
+casper.options.viewportSize = {width: 1024, height: 768};
+
+casper.test.begin('Open clarify', 4, function suite(test) {
+	casper.start(url).then(function() {
+        var openClarifyLink = '.source_clarify-in-spec_link';
+
+        this.waitForSelector(openClarifyLink,
+            function pass() {
+                this.click(openClarifyLink);
+            },
+            function fail() {
+                test.fail('Failed openning clarify link.');
+            }
+        );
+
+    }).then(function(){
+         this.waitFor(
+            function check() {
+                return (this.getCurrentUrl() === url + '?clarify=true&sections=1');
+            },
+            function then() { // step to execute when check() is ok
+                this.waitForSelector('.source_clarify_panel',
+                    function pass() {
+                        test.assertEquals(this.getCurrentUrl(), url + '?clarify=true&sections=1', 'Clarify url is corrent');
+                        test.assertExists('.source_clarify_panel', 'Clarify panel exists');
+                        test.assertExists('.source_example', 'Example exists');
+                        test.assertEval(function () {
+                            return document.querySelectorAll('.source_clarify_panel_sections option').length > 2;
+                        }, 'Should show more than 2 sections in list');
+                    },
+                    function fail() {
+                        test.fail('Failed searching for clarify link');
+                    }
+                );
+            },
+            function timeout() { // step to execute if check has failed
+                this.echo('Failed to open clarify');
+            }
+        );
+    }).run(function() { test.done() }).clear();
+});
\ No newline at end of file
diff --git a/test/unit/middlewares/clarifySpec.js b/test/unit/middlewares/clarifySpec.js
index e1fc221..65dc217 100644
--- a/test/unit/middlewares/clarifySpec.js
+++ b/test/unit/middlewares/clarifySpec.js
@@ -1,10 +1,8 @@
-var fs = require('fs');
 var path = require('path');
 
 var pathToMasterApp = path.resolve('./');
 
 var should = require('should');
-var assert = require('assert');
 var request = require('supertest');
 
 var cheerio = require('cheerio');
@@ -13,9 +11,9 @@ var loadOptions = require(path.join(pathToMasterApp, 'core/loadOptions'));
 global.opts = loadOptions(path.resolve(pathToMasterApp));
 
 
-describe.only('Clarify test /docs/spec?clarify=true', function () {
+describe('Clarify test /docs/spec?clarify=true', function () {
     describe('GET /docs/spec?clarify=true...', function () {
-        var url = 'http://localhost:8080/docs/spec/?clarify=true';
+        var url = 'http://127.0.0.1:8080/docs/spec/?clarify=true';
         it('should return nothing (&sections=77)', function (done) {
             request(url)
                 .get('&sections=77')
@@ -63,8 +61,13 @@ describe.only('Clarify test /docs/spec?clarify=true', function () {
                     }
 
                     var $ = cheerio.load(res.text, {decodeEntities: false});
+
                     var examples = $('.source_example');
+                    var headers = $('.source_section_h');
+
                     examples.length.should.equal(1);
+                    headers.length.should.equal(1);
+
                     done();
                 });
         });
@@ -102,6 +105,27 @@ describe.only('Clarify test /docs/spec?clarify=true', function () {
 
                     $('.__clear').length.should.equal(1);
                     examples.length.should.equal(0);
+
+                    done();
+                });
+        });
+
+        it('should disable javascript', function (done) {
+            request(url)
+                .get('&nojs=true')
+                .expect(200)
+                .end(function (err, res) {
+                    if (err) {
+                        throw err;
+                    }
+
+                    var $ = cheerio.load(res.text, {decodeEntities: false});
+
+                    var scripts = $('body script');
+
+                    // Only clarify script should remain
+                    scripts.length.should.be.equal(1);
+
                     done();
                 });
         });

From 83994e1fbc313e6919be4da923e9240ac4820c77 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sat, 3 Oct 2015 20:04:22 +0200
Subject: [PATCH 14/47] fix windows paths"

---
 core/middlewares/wrap.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/middlewares/wrap.js b/core/middlewares/wrap.js
index a2d6cf5..49ce273 100644
--- a/core/middlewares/wrap.js
+++ b/core/middlewares/wrap.js
@@ -43,7 +43,7 @@ exports.process = function (req, res, next) {
         // Check if we're working with included dirs
         if (context) {
             global.opts.core.common.includedDirs.forEach(function (includedDir) {
-                var pathToCheck = path.join(global.app.get('user'), includedDir) + '/';
+                var pathToCheck = (path.join(global.app.get('user'), includedDir) + '/').replace(/\\/g, '/');
 
                 if (context.indexOf(pathToCheck) === 0) {
                     context = context.replace(pathToCheck, path.join(global.pathToApp, includedDir) + '/');

From 907386afcee61f59ab71128a064d2705e63fa1a0 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 12:33:39 +0200
Subject: [PATCH 15/47] #186: remove html parser from core

---
 app.js                                      |  10 -
 assets/js/clarify.js                        |   2 -
 assets/templates/clarifyPanel.inc.html      |   7 -
 core/html-tree/html-parser/index.js         | 197 --------------------
 core/html-tree/html-parser/phantomRunner.js |  96 ----------
 core/middlewares/clarify.js                 |  38 +---
 options.js                                  |   4 -
 package.json                                |   2 -
 test/unit/middlewares/clarifySpec.js        |  79 --------
 9 files changed, 2 insertions(+), 433 deletions(-)
 delete mode 100644 core/html-tree/html-parser/index.js
 delete mode 100644 core/html-tree/html-parser/phantomRunner.js

diff --git a/app.js b/app.js
index f8a626f..695c0ac 100644
--- a/app.js
+++ b/app.js
@@ -30,7 +30,6 @@ commander
     .option('-l, --log [string]', 'Log level (default: ' + global.opts.core.common.defaultLogLevel + ').',  global.opts.core.common.defaultLogLevel)
     .option('-p, --port [number]', 'Server port (default: ' + global.opts.core.server.port + ').')
     .option('--hostname [string]', 'Server hostname  (default: ' + global.opts.core.server.hostname + ').')
-    .option('--html', 'Turn on HTML parser on app start (requires installed and enabled parser).')
     .option('--test', 'Run app with tests.')
     .option('--no-watch', 'Run with disabled watcher.')
     .option('--post-grunt [string]', 'Define Grunt command to run after app start', 'ci-post-run')
@@ -53,15 +52,6 @@ var logger = require('./core/logger');
 var log = logger.log;
 global.log = log;
 
-if (commander.html) {
-    trackStats.event({
-        group: 'features',
-        event: 'enabled html parser'
-    });
-
-    global.opts.plugins.htmlParser.enabled = true;
-    global.opts.plugins.htmlParser.onStart = true;
-}
 if (commander.port) global.opts.core.server.port = parseInt(commander.port);
 if (commander.hostname) global.opts.core.server.hostname = commander.hostname;
 if (!commander.watch) {
diff --git a/assets/js/clarify.js b/assets/js/clarify.js
index 69f2ed0..ca8a45b 100644
--- a/assets/js/clarify.js
+++ b/assets/js/clarify.js
@@ -15,10 +15,8 @@ require([
     // If we have data from Clarify output
     if (window.sourceClarifyData){
         var _options = JSON.parse(options);
-        var htmlParser = _options.plugins && _options.plugins.htmlParser && _options.plugins.htmlParser.enabled;
 
         var $panelTemplate = $(_.template(clarifyPanelTpl, {
-            htmlParser: htmlParser,
             showApiTargetOption: window.sourceClarifyData.showApiTargetOption,
             specUrl: window.sourceClarifyData.specUrl,
             tplList: window.sourceClarifyData.tplList,
diff --git a/assets/templates/clarifyPanel.inc.html b/assets/templates/clarifyPanel.inc.html
index 64000e2..6934adf 100644
--- a/assets/templates/clarifyPanel.inc.html
+++ b/assets/templates/clarifyPanel.inc.html
@@ -28,13 +28,6 @@ <h2 class="source_clarify_panel_h2">Options</h2>
                         </label>
                     </li>
                 <% } %>
-                <% if (htmlParser) { %>
-                    <li class="source_clarify_panel_option-item">
-                        <label>
-                            <input type="checkbox" class="source_clarify_panel_option-checkbox js-source_clarify_panel_option-checkbox" name="apiUpdate"> Update Spec HTML API on each Clarify call
-                        </label>
-                    </li>
-                <% } %>
                 <li class="source_clarify_panel_option-item">
                     <label>
                         <input type="checkbox" class="source_clarify_panel_option-checkbox js-source_clarify_panel_option-checkbox" name="nojs"> Remove JS scripts
diff --git a/core/html-tree/html-parser/index.js b/core/html-tree/html-parser/index.js
deleted file mode 100644
index 9797087..0000000
--- a/core/html-tree/html-parser/index.js
+++ /dev/null
@@ -1,197 +0,0 @@
-'use strict';
-
-var path = require('path');
-var async = require('async');
-var ParseData = require(path.join(global.pathToApp,'core/lib/parseData'));
-var phantom = require('phantomjs');
-var unflatten = require(path.join(global.pathToApp,'core/unflat'));
-var childProcess = require('child_process');
-var htmlTree = require(path.join(global.pathToApp,'core/html-tree'));
-var utils = require(path.join(global.pathToApp,'core/lib/utils'));
-
-var processFlagNotExec = true;
-
-var config = {
-    enabled: true,
-
-    // Run HTML parser on app start
-    onStart: false,
-    cron: false,
-    cronProd: true,
-    cronRepeatTime: 600000,
-
-    // PhantomJS retry limit
-    errorLimit: 2,
-    asyncPhantomCallLimit: 5,
-
-    specsFilter: {
-        filterOut: {
-            cats: ['docs'],
-            tags: ['parse-problems']
-        }
-    },
-
-    // Path to HTML data otput
-    pathToSpecs: path.join(global.pathToApp, global.opts.core.api.specsData)
-};
-
-// Overwriting base options
-if (global.opts.core.parseHTML) utils.extendOptions(config, global.opts.core.parseHTML); // Legacy support
-if (global.opts.plugins && global.opts.plugins.htmlParser) utils.extendOptions(config, global.opts.plugins.htmlParser);
-
-/**
- * Get list of specs for parsing with PhantomJS
- *
- * @returns {Array} Returns array with spec URLs
- */
-var getSpecsList = function() {
-    var parseSpecs = new ParseData({
-        scope: 'specs',
-        path: require.resolve(config.pathToSpecs)
-    });
-
-    var specs = parseSpecs.getFilteredData(config.specsFilter, true);
-
-    // Preparing data for specs iteration
-    specs = specs.map(function(item){
-        return item.url.substring(1);
-    });
-
-    return specs;
-};
-
-/**
- * PhantomJS async runner, calls writeDataFile on finish
- *
- * @param {Array} specs - array with URL list, that will be passed to PhantomJS
- *
- * @param {Function} [callback] - callback function
- * @param {Object} callback.err - Error object
- * @param {Object} callback.outputData - Passes output data to callback
- */
-var processSpecs = module.exports.processSpecs = function(specs, callback){
-    callback = typeof callback === 'function' ? callback : function(){};
-
-    if (!config.enabled) {
-        callback('HTML parser disabled.');
-
-        return;
-    }
-
-    if (processFlagNotExec) {
-        global.log.info('HTML API update started');
-
-        var _specs = specs || getSpecsList();
-        var specsLeft = _specs.slice(0);
-        var PhantomPath = phantom.path;
-        var outputHTML = {};
-        var errorCounter = {};
-        var specLength = _specs.length;
-        var doneCounter = 0;
-        var phExecCommand = PhantomPath + " " + path.join(global.pathToApp, 'core/html-tree/html-parser/phantomRunner.js');
-
-        processFlagNotExec = false;
-
-        global.log.trace('Processing ' + specLength + ' specs.');
-
-        async.mapLimit(_specs, config.asyncPhantomCallLimit, function (spec, next) {
-            var n = _specs.indexOf(spec) + 1;
-
-            global.log.trace('Starts...' + n, spec);
-
-            childProcess.exec(phExecCommand + " " + spec + " " + global.opts.core.server.port, function (error, stdout, stderr) {
-                handler(error, stdout, stderr, spec);
-                next();
-            });
-        });
-
-        var handler = function(error, stdout, stderr, spec) {
-            if (error) {
-                if (typeof errorCounter[spec] !== 'number') {
-                     errorCounter[spec] = 0;
-                }
-
-                errorCounter[spec]++;
-
-                // If limit is not reached, try again
-                if (errorCounter[spec] <= config.errorLimit) {
-                    global.log.debug('Rerun', spec);
-
-                    childProcess.exec(phExecCommand + " " + spec, function (error, stdout, stderr) {
-                        handler(error, stdout, stderr, spec, writeCallback);
-                    });
-                    return;
-                }
-
-                global.log.error('Exec error on spec ' + spec + ': '+ error);
-                global.log.debug('Error info: ', JSON.stringify({
-                    spec: spec,
-                    error: error,
-                    stdount: stdout,
-                    stderr: stderr
-                }));
-            } else {
-                var parsedStdout = [];
-
-                try {
-                    parsedStdout = JSON.parse(stdout);
-                } catch(e) {
-                    global.log.debug('HTML Parser stdout parse error: ', e, stdout);
-                    global.log.debug('Error from Phantom parser: ', stdout);
-                    parsedStdout = {
-                        message: "Stdout parse error"
-                    };
-                }
-
-                global.log.debug('Spec done: ', JSON.stringify({
-                    spec: spec,
-                    error: error,
-                    stderr: stderr
-                }));
-
-                // Writing contents to common obj
-                outputHTML[spec+'/specFile/contents'] = parsedStdout.contents;
-                outputHTML[spec+'/specFile/headResources'] = parsedStdout.headResources;
-                outputHTML[spec+'/specFile/bodyResources'] = parsedStdout.bodyResources;
-            }
-
-            global.log.debug((doneCounter/specLength*100).toFixed(2),'%...Done', spec);
-
-            // Logging specs queen
-            specsLeft.splice(specsLeft.indexOf(spec), 1);
-            if (specsLeft.length < 5 && specsLeft.length !== 0) {
-                global.log.trace('Specs queen', specsLeft);
-            }
-
-            doneCounter++;
-
-            // We handled all requested specs
-            if (doneCounter === specLength) {
-                var outputData = unflatten(outputHTML, { delimiter: '/', overwrite: 'root' });
-
-                // Callback is passed to writeDataFile
-                var writeCallback = function() {
-                    global.log.info('HTML API successfully updated');
-                    processFlagNotExec = true;
-
-                    callback(null, outputData);
-                };
-
-                htmlTree.writeDataFile(outputData, true, false, writeCallback);
-            }
-        };
-    }
-};
-
-if (config.enabled) {
-    // Running processSpecs by cron
-    if (config.cron || (global.MODE === 'production' && config.cronProd)) {
-        setInterval(function () {
-            processSpecs();
-        }, config.cronRepeatTime);
-    }
-
-    if (config.onStart) {
-        setTimeout(processSpecs, 100);
-    }
-}
\ No newline at end of file
diff --git a/core/html-tree/html-parser/phantomRunner.js b/core/html-tree/html-parser/phantomRunner.js
deleted file mode 100644
index 60fb8c9..0000000
--- a/core/html-tree/html-parser/phantomRunner.js
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- *
- * This script is executed in separate process from main app and runs in PhantomJS context
- *
- * */
-
-'use strict';
-
-/* global phantom: true */
-var page = require('webpage').create();
-var system = require('system');
-
-// arguments from node query
-var url = system.args[1];
-var port = system.args[2];
-
-page.onResourceReceived = function(response) {
-    if (response.id === 1 && response.status === 404 || response.status === 500) {
-        console.log(JSON.stringify({
-                "error": "Network error status "+ response.status,
-                "url": url
-            })
-        );
-        phantom.exit();
-    }
-};
-
-page.onConsoleMessage = function(msg) {
-    //console.log('-- webkit console: ' + msg);
-};
-
-page.open('http://127.0.0.1:' + port + '/' + url, function (status) {
-    if (status !== 'success') {
-        console.log(JSON.stringify({
-                "error": "Error loading page.",
-                "url": url
-            })
-        );
-
-        phantom.exit();
-    }
-
-    setTimeout(function () {
-        console.log(JSON.stringify([{
-                "error": "Too long execution time.",
-                "url": url
-            }])
-        );
-        phantom.exit();
-    }, 5000);
-});
-
-page.onCallback = function (data) {
-    if (data.message) {
-        var code = page.evaluate(function (url) {
-            var output = {};
-
-            $.ajax({
-                url: '/source/assets/js/modules/sectionsParser.js',
-                dataType: "script",
-                async: false,
-                success: function(){
-                    var parser = new SourceGetSections();
-
-                    output = parser.getSpecFull();
-                }
-            });
-
-            return output;
-        }, url);
-
-        // TODO: make reponse in {{ ... }} to parse only relevant part
-        // Returns stdout, that is then parsed from main app
-        console.log(JSON.stringify(code));
-        phantom.exit();
-
-    } else {
-        console.log("No callback received", JSON.stringify({
-                "url": url
-            })
-        );
-        phantom.exit();
-    }
-};
-
-page.onError = function(msg, trace) {
-    var log = {
-        "error": "Error onpage",
-        "message": msg,
-        "file": trace[0],
-        "line": trace[0].line,
-        "function": trace[0].function
-    };
-
-    console.log('Phantom-runner error: ', JSON.stringify(log, null, 4));
-};
\ No newline at end of file
diff --git a/core/middlewares/clarify.js b/core/middlewares/clarify.js
index 4838d0c..7690978 100644
--- a/core/middlewares/clarify.js
+++ b/core/middlewares/clarify.js
@@ -13,8 +13,6 @@ var pathToApp = path.dirname(require.main.filename);
 var specUtils = require(path.join(pathToApp, 'core/lib/specUtils'));
 var parseData = require(path.join(pathToApp, 'core/lib/parseData'));
 var specsParser = require(path.join(pathToApp, 'core/lib/specPageParser'));
-var htmlParser = require(path.join(pathToApp, 'core/html-tree/html-parser'));
-
 
 var htmlDataPath = path.join(pathToApp, global.opts.core.api.htmlData);
 var parseHTMLData = new parseData({
@@ -22,8 +20,6 @@ var parseHTMLData = new parseData({
     path: htmlDataPath
 });
 
-var htmlParserEnabled = global.opts.plugins && global.opts.plugins.htmlParser && global.opts.plugins.htmlParser.enabled;
-
 //TODO JSdoc
 
 var getTpl = function(tpl) {
@@ -155,21 +151,6 @@ var parseSpec = function(sections, pathToSpec) {
     return deferred.promise;
 };
 
-var updateApiData = function(specID) {
-    var deferred = Q.defer();
-    var specs = [specID];
-
-    htmlParser.processSpecs(specs, function(err){
-        if (err) {
-            deferred.reject(err);
-        } else {
-            deferred.resolve();
-        }
-    });
-
-    return deferred.promise;
-};
-
 var getDataFromApi = function(sections, specID, apiUpdate) {
     var deferred = Q.defer();
     var output = {};
@@ -198,20 +179,7 @@ var getDataFromApi = function(sections, specID, apiUpdate) {
         }
     };
 
-    if (apiUpdate) {
-        updateApiData(specID).then(function(){
-            getSpecData();
-        }).fail(function(err) {
-            var msg = 'Failed updating HTML Spec API. ';
-
-            deferred.reject({
-                err: err,
-                msg: msg
-            });
-        });
-    } else {
-        getSpecData();
-    }
+    getSpecData();
 
     return deferred.promise;
 };
@@ -274,8 +242,6 @@ module.exports.process = function(req, res, next) {
             return;
         }
 
-        if (!specHasHTMLAPIData) apiUpdate = true;
-
         var getSpecData = function(){
             return fromApi ? getDataFromApi(sections, specID, apiUpdate) : parseSpec(sections, parsedPath.pathToSpec);
         };
@@ -297,7 +263,7 @@ module.exports.process = function(req, res, next) {
                 };
 
                 var clarifyData = '<script>var sourceClarifyData = '+ JSON.stringify({
-                    showApiTargetOption: specHasHTMLAPIData || htmlParserEnabled,
+                    showApiTargetOption: specHasHTMLAPIData,
                     specUrl: specInfo.url,
                     sectionsIDList: getSectionsIDList(_specData.allContents),
                     tplList: tplList
diff --git a/options.js b/options.js
index e652798..b91a346 100644
--- a/options.js
+++ b/options.js
@@ -171,10 +171,6 @@ module.exports = {
 
     // External plugins options (are also exposed to client-side
     plugins: {
-        // PhantomJS HTML API parser will be moved to plugin at v.0.6
-        htmlParser: {
-            enabled: false
-        }
     },
 
     /*
diff --git a/package.json b/package.json
index 7dbff8a..1b2b45a 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,6 @@
     "url": "https://github.com/sourcejs/Source.git"
   },
   "dependencies": {
-    "async": "~0.9.0",
     "body-parser": "~1.6.4",
     "cheerio": "^0.19.0",
     "colors": "0.6.x",
@@ -51,7 +50,6 @@
     "macaddress": "^0.2.8",
     "marked": "^0.3.2",
     "path": "^0.4.9",
-    "phantomjs": "1.9.7-15",
     "pretty-hrtime": "^1.0.0",
     "q": "^1.1.1",
     "serve-favicon": "^2.2.0",
diff --git a/test/unit/middlewares/clarifySpec.js b/test/unit/middlewares/clarifySpec.js
index 65dc217..b1d2019 100644
--- a/test/unit/middlewares/clarifySpec.js
+++ b/test/unit/middlewares/clarifySpec.js
@@ -150,84 +150,5 @@ describe('Clarify test /docs/spec?clarify=true', function () {
                     done();
                 });
         });
-
-        if (global.opts.plugins && global.opts.plugins.htmlParser && global.opts.plugins.htmlParser.enabled) {
-            it('should return all sections', function (done) {
-                request(urlFromApi)
-                    .get()
-                    .expect(200)
-                    .end(function (err, res) {
-                        if (err) {
-                            throw err;
-                        }
-
-                        var $ = cheerio.load(res.text, {decodeEntities: false});
-                        var sectionHeaders = $('.source_section_h');
-                        var examples = $('.source_example');
-
-                        sectionHeaders.length.should.be.greaterThan(1);
-                        examples.length.should.be.greaterThan(0);
-
-                        done();
-                    });
-            });
-
-            it('should return one example', function (done) {
-                request(urlFromApi)
-                    .get('&sections=1.1')
-                    .expect(200)
-                    .end(function (err, res) {
-                        if (err) {
-                            throw err;
-                        }
-
-                        var $ = cheerio.load(res.text, {decodeEntities: false});
-                        var examples = $('.source_example');
-
-                        examples.length.should.equal(1);
-
-                        done();
-                    });
-            });
-
-            it('should have injected resources', function (done) {
-                request(urlFromApi)
-                    .get('&sections=1.1')
-                    .expect(200)
-                    .end(function (err, res) {
-                        if (err) {
-                            throw err;
-                        }
-
-                        var $ = cheerio.load(res.text, {decodeEntities: false});
-
-                        $('[href*="bootstrap.css"]').length.should.equal(1);
-                        $('body > style').length.should.be.greaterThan(0);
-                        $('body > script').length.should.be.greaterThan(0);
-
-                        done();
-                    });
-            });
-
-            it('should change template', function (done) {
-                request(urlFromApi)
-                    .get('&tpl=clear')
-                    .expect(200)
-                    .end(function (err, res) {
-                        if (err) {
-                            throw err;
-                        }
-
-                        var $ = cheerio.load(res.text, {decodeEntities: false});
-
-                        var examples = $('.source_example');
-
-                        $('.__clear').length.should.equal(1);
-                        examples.length.should.equal(0);
-
-                        done();
-                    });
-            });
-        }
     });
 });
\ No newline at end of file

From ff1918ee44fa3db5247eec11d266056bc9fe348e Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 12:34:26 +0200
Subject: [PATCH 16/47] cleanup npm module build

---
 .npmignore   |  4 ++++
 package.json | 14 +++++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)
 create mode 100644 .npmignore

diff --git a/.npmignore b/.npmignore
new file mode 100644
index 0000000..60161c2
--- /dev/null
+++ b/.npmignore
@@ -0,0 +1,4 @@
+test
+jsdoc
+log
+user
\ No newline at end of file
diff --git a/package.json b/package.json
index 1b2b45a..df36d93 100644
--- a/package.json
+++ b/package.json
@@ -34,13 +34,7 @@
     "grunt-cli": "^0.1.13",
     "grunt-contrib-clean": "^0.6.0",
     "grunt-contrib-copy": "~0.5.0",
-    "grunt-contrib-cssmin": "~0.6.2",
-    "grunt-contrib-htmlmin": "~0.3.0",
-    "grunt-contrib-jshint": "^0.11.2",
     "grunt-contrib-less": "~0.11.0",
-    "grunt-contrib-uglify": "~0.2.7",
-    "grunt-contrib-watch": "~0.5.3",
-    "grunt-jsdoc": "^0.5.7",
     "grunt-newer": "~0.6.1",
     "gulp-watch": "operatino/gulp-watch#4.3.5-latest-fsevents",
     "load-grunt-parent-tasks": "^0.1.1",
@@ -79,6 +73,12 @@
     "grunt-mocha-test": "^0.12.4",
     "mocha": "^2.0.1",
     "should": "~4.0.4",
-    "supertest": "~0.13.0"
+    "supertest": "~0.13.0",
+    "grunt-jsdoc": "^0.5.7",
+    "grunt-contrib-uglify": "~0.2.7",
+    "grunt-contrib-cssmin": "~0.6.2",
+    "grunt-contrib-htmlmin": "~0.3.0",
+    "grunt-contrib-jshint": "^0.11.2",
+    "grunt-contrib-watch": "~0.5.3"
   }
 }

From 0b2b71a7740e462d3983f0e868df15688daeee74 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 12:40:38 +0200
Subject: [PATCH 17/47] fix npm version check

---
 Gruntfile.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gruntfile.js b/Gruntfile.js
index 8fb2f0d..8dcf84f 100755
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -12,7 +12,7 @@ var getLoaderPackageName = function() {
     var packageName;
     var parentFolderName = path.basename(path.resolve('..'));
     var isSubPackage = parentFolderName === 'node_modules';
-    var isLocalDepsAvailable = fs.existsSync('node_modules/grunt-autoprefixer') && fs.existsSync('node_modules/grunt-contrib-cssmin');
+    var isLocalDepsAvailable = fs.existsSync('node_modules/grunt-autoprefixer') && fs.existsSync('node_modules/grunt-contrib-copy');
 
     if (isSubPackage && !isLocalDepsAvailable) {
         packageName = 'load-grunt-parent-tasks';

From b622840a52012e80349bfaf5f59a50760522c4ec Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 13:08:09 +0200
Subject: [PATCH 18/47] correct npm ignore list

---
 .npmignore | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.npmignore b/.npmignore
index 60161c2..e2f64a3 100644
--- a/.npmignore
+++ b/.npmignore
@@ -1,4 +1,4 @@
-test
-jsdoc
-log
-user
\ No newline at end of file
+test/
+jsdoc/
+log/
+user/
\ No newline at end of file

From 59a3795db657d3987060b09e253668e3dad030ae Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 13:14:40 +0200
Subject: [PATCH 19/47] correct npm ignore list

---
 .npmignore | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/.npmignore b/.npmignore
index e2f64a3..9349107 100644
--- a/.npmignore
+++ b/.npmignore
@@ -1,4 +1,24 @@
-test/
-jsdoc/
-log/
-user/
\ No newline at end of file
+*.idea
+*.iml
+
+.travis.yml
+.jshintrc
+appveyor.yml
+source.sh
+
+# Generated data for API
+/core/api/data/
+/log/
+
+# Generated JSdoc
+/jsdoc/
+
+# Build folder for assets
+/build/
+
+# Web server root, and your specs home
+# You can store other repository in this path
+/user/
+
+# Folder with tests
+/test/
\ No newline at end of file

From ddbd81dfb3543b82752c3c2bad08f90a7ff5a4ec Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 15:51:49 +0200
Subject: [PATCH 20/47] #60: improving support for sourcejs as npm module

---
 assets/js/clarify.js        |  2 --
 core/file-tree/index.js     | 11 +++++++----
 core/lib/parseData.js       |  3 +++
 core/lib/specUtils.js       |  9 ++++++---
 core/middlewares/clarify.js |  3 ++-
 core/postInstall.js         |  4 +---
 package.json                |  2 +-
 7 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/assets/js/clarify.js b/assets/js/clarify.js
index ca8a45b..1780d5a 100644
--- a/assets/js/clarify.js
+++ b/assets/js/clarify.js
@@ -14,8 +14,6 @@ require([
 
     // If we have data from Clarify output
     if (window.sourceClarifyData){
-        var _options = JSON.parse(options);
-
         var $panelTemplate = $(_.template(clarifyPanelTpl, {
             showApiTargetOption: window.sourceClarifyData.showApiTargetOption,
             specUrl: window.sourceClarifyData.specUrl,
diff --git a/core/file-tree/index.js b/core/file-tree/index.js
index cb304bf..781cd6e 100644
--- a/core/file-tree/index.js
+++ b/core/file-tree/index.js
@@ -65,10 +65,13 @@ var getRelativeSpecPath = module.exports.getRelativeSpecPath = function(absolute
         relativeSpecPath = absolutePath.replace(config.specsRoot, '');
     } else {
         // Cleaning path for included folders
-
         relativeSpecPath = absolutePath.replace(normalizedPathToApp, '');
     }
 
+    if (relativeSpecPath === '') {
+        relativeSpecPath = '/';
+    }
+
     return relativeSpecPath;
 };
 
@@ -170,7 +173,7 @@ var getSpecMeta = module.exports.getSpecMeta = function(specDirOrPath){
     var relativeSpecPath = specLocation.relativeSpecPath;
 
     // Remove first slash for ID
-    page.id = relativeSpecPath.substring(1);
+    page.id = relativeSpecPath === '/' ? '/' : relativeSpecPath.substring(1);
     page.url = relativeSpecPath;
 
     // If we have Spec, get additional meta
@@ -211,7 +214,7 @@ var fileTree = function (workingDir) {
         dirContent[i] = path.join(processingDir, dirContent[i].replace(/\\/g, '/'));
     }
 
-    //on first call we add includedDirs
+    // On first call we add includedDirs
     if (processingDir === config.specsRoot) {
         config.includedDirs.map(function (includedDir) {
             dirContent.push(path.join(normalizedPathToApp, includedDir));
@@ -220,7 +223,7 @@ var fileTree = function (workingDir) {
 
     dirContent.forEach(function(pathToFile) {
         // Path is excluded
-        if (excludes.test(processingDir)) return;
+        if (excludes.test(processingDir) || !fs.existsSync(pathToFile)) return;
 
         var infoFileName = coreOpts.common.infoFile;
         var targetFile = path.basename(pathToFile);
diff --git a/core/lib/parseData.js b/core/lib/parseData.js
index 8b731bc..12bf912 100644
--- a/core/lib/parseData.js
+++ b/core/lib/parseData.js
@@ -247,6 +247,9 @@ ParseData.prototype.getFilteredData = function(filterConf, array, data) {
  * @returns {Object} Return single object by requested ID, null (no id found) or undefined (no data found)
  */
 ParseData.prototype.getByID = function(id) {
+    // TODO: fix flat id's list, to remove this hack
+    if (id === '/') id = 'specFile';
+
     var dataExists = this.updateData();
     var targetData = this.data[id];
 
diff --git a/core/lib/specUtils.js b/core/lib/specUtils.js
index cb261a5..093bfa4 100644
--- a/core/lib/specUtils.js
+++ b/core/lib/specUtils.js
@@ -39,9 +39,11 @@ var parseSpecUrlPath = module.exports.parseSpecUrlPath = function(urlPath){
  * @returns {String} Return a parsed Spec ID
  */
 var getSpecIDFromUrl = module.exports.getSpecIDFromUrl = function(urlToSpec){
-    urlToSpec = urlToSpec.replace(/^\//, '').replace(/\/+$/, '');
-
-    return urlToSpec;
+    if (urlToSpec === '/') {
+        return urlToSpec;
+    } else {
+        return urlToSpec.replace(/^\//, '').replace(/\/+$/, '');
+    }
 };
 
 /**
@@ -57,6 +59,7 @@ module.exports.getSpecInfo = function(urlSpecPath) {
         scope: 'specs',
         path: specsDataPath
     });
+
     var specID = getSpecIDFromUrl(urlSpecPath);
 
     return parseSpecData.getByID(specID);
diff --git a/core/middlewares/clarify.js b/core/middlewares/clarify.js
index 7690978..d46341e 100644
--- a/core/middlewares/clarify.js
+++ b/core/middlewares/clarify.js
@@ -104,7 +104,8 @@ var parseSpec = function(sections, pathToSpec) {
         port: global.opts.core.server.port,
         path: pathToSpec + '?internal=true'
     };
-    var fullUrl = url.resolve(options.host + ':' + options.port, options.path);
+
+    var fullUrl = options.host + ':' + options.port + options.path;
 
     var callback = function (response) {
         var specHTML = '';
diff --git a/core/postInstall.js b/core/postInstall.js
index b338d84..7b04c8c 100644
--- a/core/postInstall.js
+++ b/core/postInstall.js
@@ -1,7 +1,6 @@
 'use strict';
 
 var path = require('path');
-var fs = require('fs');
 var link = require('./lib/createLink');
 
 var pathToApp = __dirname.replace(/^\w:\\/, function (match) {
@@ -15,8 +14,7 @@ var parentNodeModules = path.join(enginePath, '../');
 
 // check if sourcejs is installed as a node_package
 if (
-    path.relative(parentNodeModules, enginePath) === 'sourcejs' &&
-    fs.existsSync(path.join(userPath, 'options.js'))
+    path.relative(parentNodeModules, enginePath) === 'sourcejs'
 ) {
     link(userPath, internalUserPath, 'dir');
     console.log('SourceJS User folder symlink created.');
diff --git a/package.json b/package.json
index df36d93..e6c507c 100644
--- a/package.json
+++ b/package.json
@@ -54,7 +54,7 @@
   },
   "scripts": {
     "postinstall": "node ./core/postInstall && grunt update && node ./core/trackInstall",
-    "build": "npm i",
+    "build": "node ./core/postInstall && grunt update",
     "lint": "grunt jshint",
     "start": "node app",
     "watch": "grunt watch-all",

From 8fff72b8ac37059d46df8682f5da93060a8c06b5 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 16:18:42 +0200
Subject: [PATCH 21/47] update npm deps

---
 core/file-tree/index.js |  2 +-
 core/html-tree/index.js |  4 ++--
 core/lib/configUtils.js |  2 +-
 package.json            | 47 ++++++++++++++++++++---------------------
 4 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/core/file-tree/index.js b/core/file-tree/index.js
index 781cd6e..f4364e4 100644
--- a/core/file-tree/index.js
+++ b/core/file-tree/index.js
@@ -286,7 +286,7 @@ var updateFileTree = module.exports.updateFileTree = function (data, unflattenDa
         }
 
         try {
-            prevData = fs.readJsonFileSync(dataStoragePath);
+            prevData = fs.readJsonSync(dataStoragePath);
         } catch (e) {
             global.log.trace('Reading initial data error: ', e);
             global.log.debug('Extending from empty object, as we do not have initial data');
diff --git a/core/html-tree/index.js b/core/html-tree/index.js
index b12563c..ceb5d31 100644
--- a/core/html-tree/index.js
+++ b/core/html-tree/index.js
@@ -111,7 +111,7 @@ var writeDataFile = module.exports.writeDataFile = function(data, extend, dataPa
             var prevData = {};
 
             try {
-                prevData = fs.readJsonFileSync(dataStoragePath);
+                prevData = fs.readJsonSync(dataStoragePath);
             } catch (e) {
                 apiLog.trace('Reading initial data error: ', e);
                 apiLog.debug('Extending from empty object, as we do not have initial data');
@@ -170,7 +170,7 @@ module.exports.deleteFromDataFile = function(dataPath, removeID, callback) {
         var pathSplit = removeID.split('/');
 
         try {
-            data = fs.readJsonFileSync(dataPath);
+            data = fs.readJsonSync(dataPath);
         } catch (e) {}
 
         if (data) {
diff --git a/core/lib/configUtils.js b/core/lib/configUtils.js
index c38012f..0a13255 100644
--- a/core/lib/configUtils.js
+++ b/core/lib/configUtils.js
@@ -180,7 +180,7 @@ module.exports.getContextOptions = function(refUrl, defaultOpts) {
     var infoOptionsKey = global.opts.core.common.infoFileOptions;
 
     // Extent context options object with info.json contents
-    contextOptions.specInfo = fs.readJsonFileSync(infoFilePath, {throws: false});
+    contextOptions.specInfo = fs.readJsonSync(infoFilePath, {throws: false});
 
     // Override local options
     if (contextOptionsEnabled && contextOptions.specInfo && contextOptions.specInfo[infoOptionsKey]) {
diff --git a/package.json b/package.json
index e6c507c..044adb6 100644
--- a/package.json
+++ b/package.json
@@ -12,43 +12,42 @@
     "url": "https://github.com/sourcejs/Source.git"
   },
   "dependencies": {
-    "body-parser": "~1.6.4",
+    "body-parser": "^1.14.1",
     "cheerio": "^0.19.0",
     "colors": "0.6.x",
     "commander": "^2.8.1",
-    "compression": "~1.0.11",
-    "cookie-parser": "~1.3.2",
-    "deep-extend": "~0.2.11",
+    "compression": "^1.6.0",
+    "cookie-parser": "^1.4.0",
+    "deep-extend": "^0.4.0",
     "device": "^0.3.2",
     "ejs": "operatino/ejs#2.3.4-sandbox-option",
     "everyauth": "^0.4.9",
-    "express": "~4.8.3",
+    "express": "^4.13.3",
     "express-session": "^1.10.2",
     "extend": "~1.2.1",
     "flat": "^1.2.1",
-    "fs-extra": "~0.11.1",
+    "fs-extra": "^0.24.0",
     "fs-finder": "operatino/node-fs-finder#fix-windows-find-up",
     "glob": "^5.0.14",
     "grunt": "~0.4.5",
-    "grunt-autoprefixer": "~1.0.0",
+    "grunt-autoprefixer": "^3.0.3",
     "grunt-cli": "^0.1.13",
     "grunt-contrib-clean": "^0.6.0",
-    "grunt-contrib-copy": "~0.5.0",
-    "grunt-contrib-less": "~0.11.0",
-    "grunt-newer": "~0.6.1",
+    "grunt-contrib-copy": "^0.8.1",
+    "grunt-contrib-less": "^1.0.1",
+    "grunt-newer": "^1.1.1",
     "gulp-watch": "operatino/gulp-watch#4.3.5-latest-fsevents",
     "load-grunt-parent-tasks": "^0.1.1",
-    "load-grunt-tasks": "~0.3.0",
+    "load-grunt-tasks": "^3.3.0",
     "lodash": "^3.10.1",
     "log4js": "~0.6.20",
     "macaddress": "^0.2.8",
     "marked": "^0.3.2",
-    "path": "^0.4.9",
     "pretty-hrtime": "^1.0.0",
     "q": "^1.1.1",
     "serve-favicon": "^2.2.0",
     "shortid": "^2.2.2",
-    "time-grunt": "~0.2.10",
+    "time-grunt": "^1.2.1",
     "tinyforever": "0.0.3",
     "universal-analytics": "^0.3.9"
   },
@@ -66,19 +65,19 @@
     "ci-test-win": "grunt ci-pre-run && node app.js --post-grunt ci-post-run-win --test --log trace --no-watch"
   },
   "devDependencies": {
-    "assert": "~1.1.1",
+    "assert": "^1.3.0",
     "chai": "^3.2.0",
-    "cross-spawn": "^0.4.0",
+    "cross-spawn": "^2.0.0",
     "grunt-casperjs": "^2.1.0",
-    "grunt-mocha-test": "^0.12.4",
+    "grunt-contrib-cssmin": "^0.14.0",
+    "grunt-contrib-htmlmin": "^0.5.0",
+    "grunt-contrib-jshint": "^0.11.3",
+    "grunt-contrib-uglify": "^0.9.2",
+    "grunt-contrib-watch": "^0.6.1",
+    "grunt-jsdoc": "^1.0.0",
+    "grunt-mocha-test": "^0.12.7",
     "mocha": "^2.0.1",
-    "should": "~4.0.4",
-    "supertest": "~0.13.0",
-    "grunt-jsdoc": "^0.5.7",
-    "grunt-contrib-uglify": "~0.2.7",
-    "grunt-contrib-cssmin": "~0.6.2",
-    "grunt-contrib-htmlmin": "~0.3.0",
-    "grunt-contrib-jshint": "^0.11.2",
-    "grunt-contrib-watch": "~0.5.3"
+    "should": "^7.1.0",
+    "supertest": "^1.1.0"
   }
 }

From b1f456d2864da0c3265ea5b7c645240335e142e1 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 16:30:56 +0200
Subject: [PATCH 22/47] rever cross-spawn update

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 044adb6..ffeaa67 100644
--- a/package.json
+++ b/package.json
@@ -67,7 +67,7 @@
   "devDependencies": {
     "assert": "^1.3.0",
     "chai": "^3.2.0",
-    "cross-spawn": "^2.0.0",
+    "cross-spawn": "^0.4.0",
     "grunt-casperjs": "^2.1.0",
     "grunt-contrib-cssmin": "^0.14.0",
     "grunt-contrib-htmlmin": "^0.5.0",

From 9447c7522e7d334ce700ef2d804eb5f86cabfb7e Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 17:46:58 +0200
Subject: [PATCH 23/47] fix linking user folder when sourcejs in npm linked

---
 core/postInstall.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/core/postInstall.js b/core/postInstall.js
index 7b04c8c..bdabab1 100644
--- a/core/postInstall.js
+++ b/core/postInstall.js
@@ -1,21 +1,21 @@
 'use strict';
 
 var path = require('path');
+var fs = require('fs-extra');
 var link = require('./lib/createLink');
 
-var pathToApp = __dirname.replace(/^\w:\\/, function (match) {
+var pathToApp = process.env.PWD.replace(/^\w:\\/, function (match) {
     return match.toLowerCase();
 });
 
-var enginePath = path.join(pathToApp, '../');
-var userPath = path.join(enginePath, '../../');
-var internalUserPath = path.join(enginePath, 'user');
-var parentNodeModules = path.join(enginePath, '../');
+var userPath = path.join(pathToApp, '../../');
+
+var internalUserPath = path.join(pathToApp, 'user');
+var parentNodeModules = path.join(pathToApp, '../');
 
 // check if sourcejs is installed as a node_package
-if (
-    path.relative(parentNodeModules, enginePath) === 'sourcejs'
-) {
+if (path.relative(parentNodeModules, pathToApp) === 'sourcejs') {
+    fs.removeSync(internalUserPath);
     link(userPath, internalUserPath, 'dir');
     console.log('SourceJS User folder symlink created.');
 }
\ No newline at end of file

From 153cb9068f8ad1fd3f9c0f863642124ca28f1aa4 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 17:50:20 +0200
Subject: [PATCH 24/47] fix windows dirname check

---
 core/postInstall.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/postInstall.js b/core/postInstall.js
index bdabab1..60ecb6f 100644
--- a/core/postInstall.js
+++ b/core/postInstall.js
@@ -4,7 +4,8 @@ var path = require('path');
 var fs = require('fs-extra');
 var link = require('./lib/createLink');
 
-var pathToApp = process.env.PWD.replace(/^\w:\\/, function (match) {
+var currentDir = process.env.PWD || __dirname;
+var pathToApp = currentDir.replace(/^\w:\\/, function (match) {
     return match.toLowerCase();
 });
 

From 6b0842abf7590140bfbd593f48005027b8266c92 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 4 Oct 2015 18:11:11 +0200
Subject: [PATCH 25/47] node 0.4 is now supported

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index ffeaa67..a77a8bf 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
   "version": "0.6.0-dev",
   "license": "MIT",
   "engines": {
-    "node": ">=0.10.0 <4.0"
+    "node": ">=0.10.0"
   },
   "repository": {
     "type": "git",

From 792b4b9bc470fd1f0ebb29b7148f61910265acb6 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 6 Oct 2015 21:31:12 +0200
Subject: [PATCH 26/47] move ejs pre-rendering into next step after read

---
 core/middlewares/loader.js       |  6 +++++
 core/middlewares/preRenderEjs.js | 46 ++++++++++++++++++++++++++++++++
 core/middlewares/read.js         | 17 ------------
 3 files changed, 52 insertions(+), 17 deletions(-)
 create mode 100644 core/middlewares/preRenderEjs.js

diff --git a/core/middlewares/loader.js b/core/middlewares/loader.js
index 786bf35..d6d143c 100644
--- a/core/middlewares/loader.js
+++ b/core/middlewares/loader.js
@@ -117,6 +117,12 @@ module.exports.process = function(app, globalOptions){
                 group: 'pre-html',
                 indexPath: path.join(appRoot, 'core/middlewares/md.js')
             },
+            preRenderEjs: {
+                enabled: true,
+                order: -2,
+                group: 'pre-html',
+                indexPath: path.join(appRoot, 'core/middlewares/preRenderEjs.js')
+            },
             mdTag: {
                 enabled: true,
                 order: 0,
diff --git a/core/middlewares/preRenderEjs.js b/core/middlewares/preRenderEjs.js
new file mode 100644
index 0000000..cea0961
--- /dev/null
+++ b/core/middlewares/preRenderEjs.js
@@ -0,0 +1,46 @@
+'use strict';
+
+var path = require('path');
+var ejs = require(path.join(global.pathToApp, 'core/ejsWithHelpers.js'));
+var specUtils = require(path.join(global.pathToApp,'core/lib/specUtils'));
+
+/**
+ * Processing pre-ejs rendering on spec request
+ *
+ * @param {object} req - Request object
+ * @param {object} res - Response object
+ * @param {function} next - The callback function
+ * */
+exports.process = function(req, res, next) {
+    // Check if spec is request and ejs pre-rendering enabled
+    if (req.specData && req.specData.renderedHtml) {
+        var specInfo = req.specData.info;
+
+        if (specInfo.noEjs) {
+            next();
+            return;
+        }
+
+        var specPath = specUtils.getFullPathToSpec(req.path);
+        var contextOptions = req.specData.contextOptions;
+        var specFiles = contextOptions.specInfo && contextOptions.specInfo.specFile ? [contextOptions.specInfo.specFile] : contextOptions.rendering.specFiles;
+        var physicalPath = specUtils.getSpecFromDir(specPath, specFiles);
+        var processedData = req.specData.renderedHtml.replace(/^\s+|\s+$/g, '');
+
+        try {
+            processedData = ejs.render(processedData, {
+                engineVersion: global.engineVersion,
+                info: req.specData.info
+            }, {
+                filename: physicalPath
+            });
+        } catch(err){
+            global.log.warn('Could not pre-render spec with EJS: ' + req.path, err);
+        }
+        req.specData.renderedHtml = processedData;
+
+        next();
+    } else {
+        next();
+    }
+};
\ No newline at end of file
diff --git a/core/middlewares/read.js b/core/middlewares/read.js
index a11ff06..d83e3ab 100644
--- a/core/middlewares/read.js
+++ b/core/middlewares/read.js
@@ -3,7 +3,6 @@
 var fs = require('fs-extra');
 var path = require('path');
 
-var ejs = require(path.join(global.pathToApp, 'core/ejsWithHelpers.js'));
 var specUtils = require(path.join(global.pathToApp,'core/lib/specUtils'));
 var configUtils = require(path.join(global.pathToApp,'core/lib/configUtils'));
 
@@ -76,22 +75,6 @@ exports.process = function(req, res, next) {
                 capitalizedExtension = specFileExtension.charAt(0).toUpperCase() + specFileExtension.slice(1);
             }
 
-            data = data.replace(/^\s+|\s+$/g, '');
-
-            // Pre-render Spec contents with EJS
-            if (!specInfo.noEjs) {
-                try {
-                    data = ejs.render(data, {
-                        engineVersion: global.engineVersion,
-                        info: specInfo
-                    }, {
-                        filename: physicalPath
-                    });
-                } catch(err){
-                    global.log.warn('Could not pre-render spec with EJS: ' + req.path, err);
-                }
-            }
-
             req.specData["is" + capitalizedExtension] = true;
             req.specData.info = specInfo; // add spec info object to request
             req.specData.contextOptions = contextOptions; // add context options to request

From dbb1d80ee731ffd820903295b8cfc87e3b3df183 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 6 Oct 2015 21:32:48 +0200
Subject: [PATCH 27/47] update default header and footer

---
 assets/templates/footer.inc.html | 13 +++++++------
 assets/templates/header.inc.html |  5 +++--
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/assets/templates/footer.inc.html b/assets/templates/footer.inc.html
index 54701a2..13408e7 100644
--- a/assets/templates/footer.inc.html
+++ b/assets/templates/footer.inc.html
@@ -2,12 +2,13 @@
 
 <footer class="source_footer">
     <div class="source_col-main">
-        <div class="source_col">
-            <h2>Catalog</h2>
-            <ul class="source_ul">
-	            <li><a class="source_a_l" href="#777">Your link</a></li>
-            </ul>
-        </div>
+        <!-- Example markup -->
+        <!--<div class="source_col">-->
+            <!--<h2>Catalog</h2>-->
+            <!--<ul class="source_ul">-->
+	            <!--<li><a class="source_a_l" href="#777">Your link</a></li>-->
+            <!--</ul>-->
+        <!--</div>-->
         <div class="source_footer_copy">
             Copyright © 2013-2015 <a href="http://sourcejs.com" class="source_a_l">Sourcejs.com</a> <span class="source_footer_version">(v.<%- engineVersion %>)</span>
         </div>
diff --git a/assets/templates/header.inc.html b/assets/templates/header.inc.html
index f97cd10..33a0708 100644
--- a/assets/templates/header.inc.html
+++ b/assets/templates/header.inc.html
@@ -10,11 +10,12 @@
         </div>
 
         <div class="source_col source_col__23">
-            <a class="source_logo" href="/">Source</a>
+            <a class="source_logo" href="/">SourceJS</a>
 
             <nav class="source_header_nav">
                 <ul class="source_header_nav_lst">
-                    <li class="source_header_nav_i"><a class="source_a_l" href="#777">Link</a></li>
+                    <!-- Example markup -->
+                    <!--<li class="source_header_nav_i"><a class="source_a_l" href="#777">Link</a></li>-->
                 </ul>
             </nav>
 

From 0d0aa389e040622846d36533476c84821b1712c6 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Mon, 19 Oct 2015 22:23:40 +0200
Subject: [PATCH 28/47] chore: update npm deps

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index a77a8bf..ee5ec03 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,7 @@
   "dependencies": {
     "body-parser": "^1.14.1",
     "cheerio": "^0.19.0",
-    "colors": "0.6.x",
+    "colors": "^1.1.2",
     "commander": "^2.8.1",
     "compression": "^1.6.0",
     "cookie-parser": "^1.4.0",

From d935e6e59cff0b2e601858cbea4ca2b3d47f5bf8 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Mon, 19 Oct 2015 22:56:36 +0200
Subject: [PATCH 29/47] chore: remove extend and deepExtend deps

---
 core/file-tree/index.js  | 7 +++----
 core/lib/processMd.js    | 4 ++--
 core/watch/childWatch.js | 4 ++--
 package.json             | 2 --
 4 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/core/file-tree/index.js b/core/file-tree/index.js
index f4364e4..3f86e14 100644
--- a/core/file-tree/index.js
+++ b/core/file-tree/index.js
@@ -1,8 +1,7 @@
 'use strict';
 
 var fs = require('fs-extra');
-var extend = require('extend');
-var deepExtend = require('deep-extend');
+var _ = require('lodash');
 var path = require('path');
 var extendTillSpec = require(path.join(global.pathToApp,'core/lib/extendTillSpec'));
 var unflatten = require(path.join(global.pathToApp,'core/unflat'));
@@ -191,7 +190,7 @@ var getSpecMeta = module.exports.getSpecMeta = function(specDirOrPath){
     page.thumbnail = getThumbnailPath(specDir, specInfo.thumbnailPath) || false;
 
     // Apply info file contents on top
-    deepExtend(page, specInfo);
+    _.merge(page, specInfo);
 
     return page;
 };
@@ -239,7 +238,7 @@ var fileTree = function (workingDir) {
             // Going deeper
             var childObj = fileTree(pathToFile);
             if (Object.getOwnPropertyNames(childObj).length !== 0) {
-                outputJSON[targetFile] = extend(outputJSON[targetFile], childObj);
+                outputJSON[targetFile] = childObj;
             }
 
         } else if (targetFile.toLowerCase() === infoFileName.toLowerCase()) {
diff --git a/core/lib/processMd.js b/core/lib/processMd.js
index d3cb8ed..4be3fb6 100644
--- a/core/lib/processMd.js
+++ b/core/lib/processMd.js
@@ -3,7 +3,7 @@
 var path = require('path');
 var marked = require('marked');
 var cheerio = require('cheerio');
-var deepExtend = require('deep-extend');
+var _ = require('lodash');
 var translit = require(path.join(global.pathToApp,'core/lib/translit'));
 var utils = require(path.join(global.pathToApp,'core/lib/utils'));
 
@@ -47,7 +47,7 @@ renderer.heading = function (text, level) {
 };
 
 // Extend re-defined renderer
-config.marked.renderer = deepExtend(renderer, config.marked.renderer);
+config.marked.renderer = _.merge(renderer, config.marked.renderer);
 
 marked.setOptions(config.marked);
 
diff --git a/core/watch/childWatch.js b/core/watch/childWatch.js
index 92d5ee5..9ff3b9c 100644
--- a/core/watch/childWatch.js
+++ b/core/watch/childWatch.js
@@ -4,7 +4,7 @@
 
 var path = require('path');
 var watch = require('gulp-watch');
-var deepExtend = require('deep-extend');
+var _ = require('lodash');
 var commander = require('commander');
 
 // Parse arguments
@@ -47,7 +47,7 @@ utils.extendOptions(config, global.opts.core.watch);
 var prepareData = function(data, targetFile){
     var dir = path.dirname(targetFile);
 
-    return deepExtend(fileTree.getSpecMeta(dir), data);
+    return _.merge(fileTree.getSpecMeta(dir), data);
 };
 
 if (config.enabled){
diff --git a/package.json b/package.json
index ee5ec03..907b46d 100644
--- a/package.json
+++ b/package.json
@@ -18,13 +18,11 @@
     "commander": "^2.8.1",
     "compression": "^1.6.0",
     "cookie-parser": "^1.4.0",
-    "deep-extend": "^0.4.0",
     "device": "^0.3.2",
     "ejs": "operatino/ejs#2.3.4-sandbox-option",
     "everyauth": "^0.4.9",
     "express": "^4.13.3",
     "express-session": "^1.10.2",
-    "extend": "~1.2.1",
     "flat": "^1.2.1",
     "fs-extra": "^0.24.0",
     "fs-finder": "operatino/node-fs-finder#fix-windows-find-up",

From 6d0348a30328e10422cbf5905590bca9cdfa9606 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Mon, 19 Oct 2015 23:02:31 +0200
Subject: [PATCH 30/47] chore: add .bithoundrc

---
 .bithoundrc | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 .bithoundrc

diff --git a/.bithoundrc b/.bithoundrc
new file mode 100644
index 0000000..b30de9a
--- /dev/null
+++ b/.bithoundrc
@@ -0,0 +1,10 @@
+{
+  "ignore": [
+    "assets/js/lib/**",
+    "assets/test/jasmine/**"
+  ],
+  "test": [
+    "test/**",
+    "assets/test/**"
+  ]
+}
\ No newline at end of file

From 300e9c7e9bb7e4654ac1b79e509864a664e2330f Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 20 Oct 2015 12:03:53 +0200
Subject: [PATCH 31/47] remove appveyor unstable settings

---
 appveyor.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index ae75ddf..dc95b4a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -23,6 +23,4 @@ matrix:
   fast_finish: true
 
 cache:
-  - C:\Users\appveyor\AppData\Roaming\npm\node_modules -> package.json  # global npm modules
-  - C:\Users\appveyor\AppData\Roaming\npm-cache -> package.json         # npm cache
-  - node_modules -> package.json                                        # local npm modules
\ No newline at end of file
+  - node_modules -> package.json # Local npm modules

From 7d6c3abca8c3fdfe7f82525412eafb0fe0d1ff8a Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 20 Oct 2015 12:05:24 +0200
Subject: [PATCH 32/47] add Node.js v4 to appveyor

---
 appveyor.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/appveyor.yml b/appveyor.yml
index dc95b4a..9d4c74b 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -5,6 +5,7 @@ environment:
   matrix:
     - nodejs_version: 0.10
     - nodejs_version: 0.12
+    - nodejs_version: 4
 
 install:
   - ps: Install-Product node $env:nodejs_version

From e047071929e982d3c484679431bf834f6edbe4ed Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Tue, 20 Oct 2015 12:06:18 +0200
Subject: [PATCH 33/47] add Node.js v4 to travis

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 494dab7..9cd6563 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,12 +1,12 @@
 language: node_js
 node_js:
   - "0.10"
-  - "0.11"
   - "0.12"
+  - "4"
 sudo: false
 notifications:
   email:
     on_success: never
     on_failure: always
 before_script: "git clone https://github.com/sourcejs/init.git user"
-script: npm run ci-test
\ No newline at end of file
+script: npm run ci-test

From ab5cbd0def7b2a7c6908528650781d55c53e4cd6 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Thu, 22 Oct 2015 21:29:05 +0200
Subject: [PATCH 34/47] hotfix: fix node v4 unit test

---
 test/unit/middlewares/clarifySpec.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/middlewares/clarifySpec.js b/test/unit/middlewares/clarifySpec.js
index b1d2019..f7745a8 100644
--- a/test/unit/middlewares/clarifySpec.js
+++ b/test/unit/middlewares/clarifySpec.js
@@ -33,7 +33,7 @@ describe('Clarify test /docs/spec?clarify=true', function () {
 
         it('should return all sections', function (done) {
             request(url)
-                .get()
+                .get('')
                 .expect(200)
                 .end(function (err, res) {
                     if (err) {

From 6481b9d92a476a0e1b35f7f310b3443c14d7f4c9 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Thu, 22 Oct 2015 21:38:53 +0200
Subject: [PATCH 35/47] clean npm cache in CI

---
 .travis.yml  | 3 +++
 appveyor.yml | 1 +
 2 files changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 9cd6563..0fbaa2e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,9 @@ node_js:
   - "0.12"
   - "4"
 sudo: false
+install:
+  - npm cache clean
+  - npm install
 notifications:
   email:
     on_success: never
diff --git a/appveyor.yml b/appveyor.yml
index 9d4c74b..c96b1ce 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -9,6 +9,7 @@ environment:
 
 install:
   - ps: Install-Product node $env:nodejs_version
+  - npm cache clean
   - npm install
 
 build: off

From 597132bb7f6f41e89a0c974e1aadd7582e223eef Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Thu, 22 Oct 2015 21:56:17 +0200
Subject: [PATCH 36/47] #190: add node module check lib

---
 core/lib/isNodeModule.js | 13 +++++++++++++
 core/postInstall.js      |  4 ++--
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 core/lib/isNodeModule.js

diff --git a/core/lib/isNodeModule.js b/core/lib/isNodeModule.js
new file mode 100644
index 0000000..c379aff
--- /dev/null
+++ b/core/lib/isNodeModule.js
@@ -0,0 +1,13 @@
+'use strict';
+
+var path = require('path');
+var fs = require('fs-extra');
+
+module.exports = function(customPathToApp) {
+    var pathToApp = customPathToApp || global.pathToApp;
+    var grantParentPath = path.join(pathToApp, '../../');
+    var relativePathToApp = path.relative(grantParentPath, pathToApp);
+    var assumedSourceNpmPath = path.join('node_modules', fs.readJsonSync(path.join(pathToApp, 'package.json')).name);
+
+    return relativePathToApp === assumedSourceNpmPath;
+};
\ No newline at end of file
diff --git a/core/postInstall.js b/core/postInstall.js
index 60ecb6f..3a77d05 100644
--- a/core/postInstall.js
+++ b/core/postInstall.js
@@ -3,6 +3,7 @@
 var path = require('path');
 var fs = require('fs-extra');
 var link = require('./lib/createLink');
+var isNodeModule = require('./lib/isNodeModule');
 
 var currentDir = process.env.PWD || __dirname;
 var pathToApp = currentDir.replace(/^\w:\\/, function (match) {
@@ -12,10 +13,9 @@ var pathToApp = currentDir.replace(/^\w:\\/, function (match) {
 var userPath = path.join(pathToApp, '../../');
 
 var internalUserPath = path.join(pathToApp, 'user');
-var parentNodeModules = path.join(pathToApp, '../');
 
 // check if sourcejs is installed as a node_package
-if (path.relative(parentNodeModules, pathToApp) === 'sourcejs') {
+if (isNodeModule(pathToApp)) {
     fs.removeSync(internalUserPath);
     link(userPath, internalUserPath, 'dir');
     console.log('SourceJS User folder symlink created.');

From 8c49e49f6bca9b6fe1af39bfe95297f848a5dabd Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Thu, 22 Oct 2015 23:14:27 +0200
Subject: [PATCH 37/47] #190: remove auto-linking and fix npm link issue with
 sourcejs dev flow

---
 app.js                  |  3 ++-
 core/lib/createLink.js  | 52 -----------------------------------------
 core/lib/getUserPath.js | 18 ++++++++++++++
 core/postInstall.js     | 22 -----------------
 options.js              |  3 ++-
 package.json            |  2 +-
 6 files changed, 23 insertions(+), 77 deletions(-)
 delete mode 100644 core/lib/createLink.js
 create mode 100644 core/lib/getUserPath.js
 delete mode 100644 core/postInstall.js

diff --git a/app.js b/app.js
index 695c0ac..7f8b06b 100644
--- a/app.js
+++ b/app.js
@@ -38,9 +38,10 @@ commander
 global.commander = commander;
 
 var trackStats = require(path.join(global.pathToApp, 'core/trackStats'));
+var userPath = (require('./core/lib/getUserPath'))();
 
 app.set('views', path.join(__dirname, 'core/views'));
-app.set('user', path.join(__dirname, global.opts.core.common.pathToUser));
+app.set('user', userPath);
 
 // We support `development` (default), `production` and `presentation` (for demos)
 var MODE = global.MODE = process.env.NODE_ENV || 'development';
diff --git a/core/lib/createLink.js b/core/lib/createLink.js
deleted file mode 100644
index 4be8dfb..0000000
--- a/core/lib/createLink.js
+++ /dev/null
@@ -1,52 +0,0 @@
-// Modified lib from Bower
-
-'use strict';
-
-var fs = require('fs-extra');
-var path = require('path');
-var Q = require('q');
-
-var isWin = process.platform === 'win32';
-
-function createLink(src, dst, type) {
-    var dstDir = path.dirname(dst);
-
-    // Create directory
-    return Q.nfcall(fs.mkdirp, dstDir)
-    // Check if source exists
-    .then(function () {
-        return Q.nfcall(fs.stat, src)
-        .fail(function (error) {
-            if (error.code === 'ENOENT') {
-                console.log('Failed to create link to ' + path.basename(src), 'ENOENT', {
-                    details: src + ' does not exist or points to a non-existent file'
-                });
-            }
-
-            throw error;
-        });
-    })
-    // Create symlink
-    .then(function (stat) {
-        type = type || (stat.isDirectory() ? 'dir' : 'file');
-
-        return Q.nfcall(fs.symlink, src, dst, type)
-        .fail(function (err) {
-            if (!isWin || err.code !== 'EPERM') {
-                throw err;
-            }
-
-            // Try with type "junction" on Windows
-            // Junctions behave equally to true symlinks and can be created in
-            // non elevated terminal (well, not always..)
-            return Q.nfcall(fs.symlink, src, dst, 'junction')
-            .fail(function (err) {
-                console.log('Unable to create link to ' + path.basename(src), err.code, {
-                    details: err.message.trim() + '\n\nTry running this command in an elevated terminal (run as root/administrator).'
-                });
-            });
-        });
-    });
-}
-
-module.exports = createLink;
\ No newline at end of file
diff --git a/core/lib/getUserPath.js b/core/lib/getUserPath.js
new file mode 100644
index 0000000..6b095ce
--- /dev/null
+++ b/core/lib/getUserPath.js
@@ -0,0 +1,18 @@
+'use strict';
+
+var path = require('path');
+var fs = require('fs');
+var pathToApp = global.pathToApp;
+var isNodeModule = require('./isNodeModule');
+
+module.exports = function(){
+    var sourcePathFromParent = path.join(process.cwd(), 'node_modules/sourcejs');
+
+    if (fs.existsSync(sourcePathFromParent)) {
+        pathToApp = sourcePathFromParent;
+    }
+
+    var relativeUserPath = isNodeModule(pathToApp) ? '../../' : global.opts.core.common.pathToUser;
+
+    return path.join(pathToApp, relativeUserPath);
+};
\ No newline at end of file
diff --git a/core/postInstall.js b/core/postInstall.js
deleted file mode 100644
index 3a77d05..0000000
--- a/core/postInstall.js
+++ /dev/null
@@ -1,22 +0,0 @@
-'use strict';
-
-var path = require('path');
-var fs = require('fs-extra');
-var link = require('./lib/createLink');
-var isNodeModule = require('./lib/isNodeModule');
-
-var currentDir = process.env.PWD || __dirname;
-var pathToApp = currentDir.replace(/^\w:\\/, function (match) {
-    return match.toLowerCase();
-});
-
-var userPath = path.join(pathToApp, '../../');
-
-var internalUserPath = path.join(pathToApp, 'user');
-
-// check if sourcejs is installed as a node_package
-if (isNodeModule(pathToApp)) {
-    fs.removeSync(internalUserPath);
-    link(userPath, internalUserPath, 'dir');
-    console.log('SourceJS User folder symlink created.');
-}
\ No newline at end of file
diff --git a/options.js b/options.js
index b91a346..37edd99 100644
--- a/options.js
+++ b/options.js
@@ -17,7 +17,8 @@ module.exports = {
             // Name of context level settings file
             contextOptionsFile: 'sourcejs-options.js',
 
-            // Path to your SourceJS configuration folder
+            // Custom path to user contents path, relative to SourceJS root
+            // Used only in case if SourceJS installed as a parent folder for content (old approach)
             pathToUser: 'user',
 
             // Name of spec meta info file
diff --git a/package.json b/package.json
index 907b46d..df5532a 100644
--- a/package.json
+++ b/package.json
@@ -50,7 +50,7 @@
     "universal-analytics": "^0.3.9"
   },
   "scripts": {
-    "postinstall": "node ./core/postInstall && grunt update && node ./core/trackInstall",
+    "postinstall": "grunt update && node ./core/trackInstall",
     "build": "node ./core/postInstall && grunt update",
     "lint": "grunt jshint",
     "start": "node app",

From 981f7ef414bf72dbf212562fd9bba282a7672f5d Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Thu, 22 Oct 2015 23:38:06 +0200
Subject: [PATCH 38/47] check travis npm mode testing

---
 .travis.yml | 1 +
 app.js      | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 0fbaa2e..5d6a3df 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,3 +13,4 @@ notifications:
     on_failure: always
 before_script: "git clone https://github.com/sourcejs/init.git user"
 script: npm run ci-test
+after_success: "mkdir user/node_modules && cp -R . user/node_modules/sourcejs && cd user/node_modules/sourcejs && npm run ci-test"
diff --git a/app.js b/app.js
index 7f8b06b..f408664 100644
--- a/app.js
+++ b/app.js
@@ -43,6 +43,9 @@ var userPath = (require('./core/lib/getUserPath'))();
 app.set('views', path.join(__dirname, 'core/views'));
 app.set('user', userPath);
 
+//tmp
+console.log('userPath', userPath);
+
 // We support `development` (default), `production` and `presentation` (for demos)
 var MODE = global.MODE = process.env.NODE_ENV || 'development';
 

From 7684a4e587dae3b473f1de745a01dba046ffa6cf Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Thu, 22 Oct 2015 23:43:39 +0200
Subject: [PATCH 39/47] once again

---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5d6a3df..71df2a3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,5 +12,6 @@ notifications:
     on_success: never
     on_failure: always
 before_script: "git clone https://github.com/sourcejs/init.git user"
-script: npm run ci-test
-after_success: "mkdir user/node_modules && cp -R . user/node_modules/sourcejs && cd user/node_modules/sourcejs && npm run ci-test"
+script:
+  - npm run ci-test
+  - "mkdir user/node_modules && cp -R . user/node_modules/sourcejs && cd user/node_modules/sourcejs && npm run ci-test"

From 8908f2278aba25e61451af1a29ae30b6a7842b65 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Thu, 22 Oct 2015 23:50:30 +0200
Subject: [PATCH 40/47] once again2

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 71df2a3..51aab8b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,4 +14,4 @@ notifications:
 before_script: "git clone https://github.com/sourcejs/init.git user"
 script:
   - npm run ci-test
-  - "mkdir user/node_modules && cp -R . user/node_modules/sourcejs && cd user/node_modules/sourcejs && npm run ci-test"
+  - "mkdir user/node_modules && mv user ../user && cd .. && mv Source user/node_modules/sourcejs && cd user/node_modules/sourcejs && npm run ci-test"

From 1f9517ef24794a635e999d87f1eaaded25025d2f Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 25 Oct 2015 15:21:15 +0100
Subject: [PATCH 41/47] #147: remove req.path override

---
 core/middlewares/read.js | 20 ++++++++------------
 core/trackStats.js       |  2 +-
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/core/middlewares/read.js b/core/middlewares/read.js
index d83e3ab..9b013d7 100644
--- a/core/middlewares/read.js
+++ b/core/middlewares/read.js
@@ -18,7 +18,7 @@ var configUtils = require(path.join(global.pathToApp,'core/lib/configUtils'));
 exports.process = function(req, res, next) {
     var reqExt = path.extname(req.path);
 
-    // Skip static resources, and dirs witout trailing slash
+    // Skip static resources, and dirs without trailing slash
     if (reqExt !== "" || (reqExt === "" && req.path.slice(-1) !== '/')) {
         next();
         return;
@@ -29,9 +29,15 @@ exports.process = function(req, res, next) {
 
     // Check if folder is requested but not the reserved namespaces
     if (!apiRe.test(req.path) && !sourceRe.test(req.path)) {
+        var specPath = specUtils.getFullPathToSpec(req.path);
+
+        if (!fs.existsSync(specPath)) {
+            next();
+            return;
+        }
+
         global.specLoadTime = process.hrtime();
 
-        var specPath = specUtils.getFullPathToSpec(req.path);
         var contextOptions = configUtils.getContextOptions(req.path);
 
         var specFiles = contextOptions.specInfo && contextOptions.specInfo.specFile ? [contextOptions.specInfo.specFile] : contextOptions.rendering.specFiles;
@@ -44,16 +50,6 @@ exports.process = function(req, res, next) {
             return;
         }
 
-        // Passing req params
-        var urlParams = req.url.split('?')[1];
-        var paramsString = urlParams ? '?' + urlParams : '';
-
-        // Modifying url and saving params string
-        // TODO: remove in next non-patch release https://github.com/sourcejs/Source/issues/147
-        req.originalUrl = req.url;
-        req.originalPath = req.path;
-        req.url = path.join(req.path, specFile) + paramsString;
-
         fs.readFile(physicalPath, 'utf8', function (err, data) {
             if (err) {
                 res.send(err);
diff --git a/core/trackStats.js b/core/trackStats.js
index 9c02c2e..0c9d959 100644
--- a/core/trackStats.js
+++ b/core/trackStats.js
@@ -124,7 +124,7 @@ module.exports.specs = function(req) {
 
     if (q.internal) return;
 
-    if (req.originalPath === '/') {
+    if (req.path === '/') {
         pageName = '/';
     } else if (req.specData && req.specData.info && req.specData.info.role === 'navigation') {
         pageName = 'navigation';

From 85b7ce9ca031261c19e470ab540400c97cc0b9da Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 25 Oct 2015 15:54:30 +0100
Subject: [PATCH 42/47] #190: user new userPath global

---
 app.js                      | 13 ++++++-------
 core/file-tree/index.js     |  5 +++--
 core/headerFooter.js        |  2 +-
 core/lib/configUtils.js     |  2 +-
 core/lib/pathResolver.js    |  4 ++--
 core/lib/specUtils.js       |  2 +-
 core/loadPlugins.js         |  2 +-
 core/middlewares/clarify.js |  4 ++--
 core/middlewares/loader.js  |  2 +-
 core/middlewares/wrap.js    |  2 +-
 core/routes/index.js        |  2 +-
 core/watch/childWatch.js    | 14 +++++++++-----
 12 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/app.js b/app.js
index 7f8b06b..9b33e1c 100644
--- a/app.js
+++ b/app.js
@@ -38,10 +38,9 @@ commander
 global.commander = commander;
 
 var trackStats = require(path.join(global.pathToApp, 'core/trackStats'));
-var userPath = (require('./core/lib/getUserPath'))();
+var userPath = global.userPath = (require('./core/lib/getUserPath'))();
 
-app.set('views', path.join(__dirname, 'core/views'));
-app.set('user', userPath);
+console.log('Running user contents from', '`' + userPath + '`.');
 
 // We support `development` (default), `production` and `presentation` (for demos)
 var MODE = global.MODE = process.env.NODE_ENV || 'development';
@@ -111,7 +110,7 @@ app.use(function (req, res, next) {
 });
 
 // Favicon
-var faviconPath = path.join(app.get('user'), 'favicon.ico');
+var faviconPath = path.join(userPath, 'favicon.ico');
 if (fs.existsSync(faviconPath)){
     app.use(favicon(faviconPath));
 }
@@ -170,7 +169,7 @@ require("./core/loadPlugins.js");
 
 try {
     // User additional functionality
-    require(app.get('user') + "/core/app.js");
+    require(userPath + "/core/app.js");
 } catch(e){}
 
 
@@ -192,7 +191,7 @@ if (global.opts.core.watch.enabled && global.MODE === 'development') {
 var headerFooter = require('./core/headerFooter');
 
 // Static content
-app.use(express.static(app.get('user')));
+app.use(express.static(userPath));
 
 // Page 404
 app.use(function(req, res){
@@ -219,7 +218,7 @@ var logErrors = function(err, req, res, next) {
         var url = req.url || '';
 
         log.debug(req.method, req.headers);
-        log.error(('Requested url: ' + url).red, ('Error: ' + err.stack).red);
+        log.error(('Error on requesting url: ' + url).red +'.', ('Error: ' + err.stack).red);
 
         if (req.xhr) {
             res.status(500).json({msg: 'Server error'});
diff --git a/core/file-tree/index.js b/core/file-tree/index.js
index 3f86e14..081b32c 100644
--- a/core/file-tree/index.js
+++ b/core/file-tree/index.js
@@ -11,6 +11,8 @@ var coreOpts = global.opts.core;
 var prettyHrtime = require('pretty-hrtime');
 
 var busy = false;
+var normalizedPathToApp = global.pathToApp.replace(/\\/g, '/');
+var userPath = global.userPath.replace(/\\/g, '/');
 
 var config = {
     includedDirs: coreOpts.common.includedDirs,
@@ -22,7 +24,7 @@ var config = {
     cronProd: true,
     cronRepeatTime: 60000,
     outputFile: path.join(global.pathToApp, 'core/api/data/pages-tree.json'),
-    specsRoot: path.join(global.pathToApp, coreOpts.common.pathToUser).replace(/\\/g, '/'),
+    specsRoot: userPath,
     busyTimeout: 300,
     thumbnail: 'thumbnail.png'
 };
@@ -30,7 +32,6 @@ var config = {
 // Overwriting base options
 utils.extendOptions(config, coreOpts.fileTree);
 
-var normalizedPathToApp = global.pathToApp.replace(/\\/g, '/');
 
 var prepareExcludesRegex = function(){
     var dirsForRegExp = '';
diff --git a/core/headerFooter.js b/core/headerFooter.js
index 1220ff0..5eb7cbf 100644
--- a/core/headerFooter.js
+++ b/core/headerFooter.js
@@ -5,7 +5,7 @@ var path = require('path');
 
 exports.getHeaderAndFooter = function () {
     var defaultTemplatePath = path.join(global.pathToApp, 'assets/templates');
-    var userTemplatePath = path.join(global.app.get('user'), 'assets/templates');
+    var userTemplatePath = path.join(global.userPath, 'assets/templates');
     var headerFile = 'header.inc.html';
     var footerFile = 'footer.inc.html';
 
diff --git a/core/lib/configUtils.js b/core/lib/configUtils.js
index 0a13255..8018df8 100644
--- a/core/lib/configUtils.js
+++ b/core/lib/configUtils.js
@@ -48,7 +48,7 @@ var getContextOptionsList = module.exports.getContextOptionsList = function(star
     var checkPath = fs.existsSync(startPath);
     if (!checkPath) return [];
 
-    var searchStopPath = global.app.get('user');
+    var searchStopPath = global.userPath;
     var fileToFind = path.sep + global.opts.core.common.contextOptionsFile;
 
     // Skip if start path is behind stop path
diff --git a/core/lib/pathResolver.js b/core/lib/pathResolver.js
index c51631d..0abfb50 100644
--- a/core/lib/pathResolver.js
+++ b/core/lib/pathResolver.js
@@ -17,8 +17,8 @@ var resolve = module.exports.resolve = function(pathToResolve, context){
 
     var map = {
         sourcejs: global.pathToApp,
-        user: global.app.get('user'),
-        context: context || global.app.get('user')
+        user: global.userPath,
+        context: context || global.userPath
     };
     var result = pathToResolve;
 
diff --git a/core/lib/specUtils.js b/core/lib/specUtils.js
index 093bfa4..42440c1 100644
--- a/core/lib/specUtils.js
+++ b/core/lib/specUtils.js
@@ -112,7 +112,7 @@ module.exports.getSpecFromDir = function(dirPath, specFiles) {
 module.exports.getFullPathToSpec = function(urlPath){
     var pathToSpec = parseSpecUrlPath(urlPath).pathToSpec;
     var cleanPath = urlPath.replace(/\/+$/, '').replace(/\//, '');
-    var specPath = path.join(global.app.get('user'), pathToSpec).replace(/\\/g, '/');
+    var specPath = path.join(global.userPath, pathToSpec).replace(/\\/g, '/');
 
     // Including non-standard paths, outside default static route
     global.opts.core.common.includedDirs.forEach(function(item){
diff --git a/core/loadPlugins.js b/core/loadPlugins.js
index 4c1f150..0dacec5 100644
--- a/core/loadPlugins.js
+++ b/core/loadPlugins.js
@@ -8,7 +8,7 @@
 
 var fs = require('fs');
 var path = require('path');
-var pathToModules = path.join(global.app.get('user'), 'node_modules');
+var pathToModules = path.join(global.userPath, 'node_modules');
 
 // Loading all sourcejs-*/core/index.js files from npm plugins section
 if (fs.existsSync(pathToModules)) {
diff --git a/core/middlewares/clarify.js b/core/middlewares/clarify.js
index d46341e..2ed8362 100644
--- a/core/middlewares/clarify.js
+++ b/core/middlewares/clarify.js
@@ -29,7 +29,7 @@ var getTpl = function(tpl) {
 
     // TODO: use view resolver
     var pathToTemplate = path.join(pathToApp, 'core/views/clarify', templateName + '.ejs');
-    var userPathToTemplate = path.join(global.app.get('user'), 'core/views/clarify', templateName + '.ejs');
+    var userPathToTemplate = path.join(global.userPath, 'core/views/clarify', templateName + '.ejs');
 
     // First we check user tempalate, then core
     fs.readFile(userPathToTemplate, 'utf-8', function(err, data){
@@ -56,7 +56,7 @@ var getTplList = function(){
     var deferred = Q.defer();
 
     var pathToTemplates = path.join(pathToApp, 'core/views/clarify');
-    var userPathToTemplates = path.join(global.app.get('user'), 'core/views/clarify');
+    var userPathToTemplates = path.join(global.userPath, 'core/views/clarify');
 
     var templatesList = [];
 
diff --git a/core/middlewares/loader.js b/core/middlewares/loader.js
index d6d143c..0545121 100644
--- a/core/middlewares/loader.js
+++ b/core/middlewares/loader.js
@@ -158,7 +158,7 @@ module.exports.process = function(app, globalOptions){
     utils.extendOptions(
         config,
         {
-            list: gatherMiddlewares(path.join(app.get('user'), 'node_modules'), new RegExp(/^sourcejs-/), 'core/middleware/index.js')
+            list: gatherMiddlewares(path.join(global.userPath, 'node_modules'), new RegExp(/^sourcejs-/), 'core/middleware/index.js')
         },
         globalOptions.core.middlewares
     );
diff --git a/core/middlewares/wrap.js b/core/middlewares/wrap.js
index 49ce273..efbc65f 100644
--- a/core/middlewares/wrap.js
+++ b/core/middlewares/wrap.js
@@ -43,7 +43,7 @@ exports.process = function (req, res, next) {
         // Check if we're working with included dirs
         if (context) {
             global.opts.core.common.includedDirs.forEach(function (includedDir) {
-                var pathToCheck = (path.join(global.app.get('user'), includedDir) + '/').replace(/\\/g, '/');
+                var pathToCheck = (path.join(global.userPath, includedDir) + '/').replace(/\\/g, '/');
 
                 if (context.indexOf(pathToCheck) === 0) {
                     context = context.replace(pathToCheck, path.join(global.pathToApp, includedDir) + '/');
diff --git a/core/routes/index.js b/core/routes/index.js
index bd07129..719fc83 100644
--- a/core/routes/index.js
+++ b/core/routes/index.js
@@ -5,5 +5,5 @@ require("./redirects.js");
 
 // User custom routes
 try {
-    require(global.app.get('user') + "/core/routes");
+    require(global.userPath + "/core/routes");
 } catch(e){}
\ No newline at end of file
diff --git a/core/watch/childWatch.js b/core/watch/childWatch.js
index 9ff3b9c..e1adcbc 100644
--- a/core/watch/childWatch.js
+++ b/core/watch/childWatch.js
@@ -25,6 +25,8 @@ var logger = require(path.join(global.pathToApp, 'core/logger'));
 var log = logger.log;
 global.log = log;
 
+var userPath = global.userPath = (require(path.join(global.pathToApp, 'core/lib/getUserPath')))();
+
 var utils = require(path.join(global.pathToApp, 'core/lib/utils'));
 var fileTree = require(path.join(global.pathToApp, 'core/file-tree'));
 
@@ -33,11 +35,13 @@ var config = {
     verbose: false,
     glob: [
         '!' + global.pathToApp + '/**/.*/**',
-        '!' + global.pathToApp + '/**/bower_components/**',
-        '!' + global.pathToApp + '/**/node_modules/**',
+        '!' + userPath + '/**/.*/**',
+        '!' + userPath + '/**/bower_components/**',
+        '!' + userPath + '/**/node_modules/**',
 
-        global.pathToApp + '/**/' + global.opts.core.common.infoFile,
-        global.pathToApp + '/' + global.opts.core.common.pathToUser + '/**/' + global.opts.core.common.infoFile
+        global.pathToApp + '/docs/**/' + global.opts.core.common.infoFile,
+        global.pathToApp + '/test/**/' + global.opts.core.common.infoFile,
+        userPath + '/**/' + global.opts.core.common.infoFile
     ],
     ignoreHiddenFiles: true
 };
@@ -60,7 +64,7 @@ if (config.enabled){
     }, function(vinyl){
         var filePath = vinyl.path.replace(/\\/g, '/');
         var event = vinyl.event;
-        var cleanPath = filePath.replace(global.pathToApp + '/', '').replace(global.opts.core.common.pathToUser +'/', '');
+        var cleanPath = filePath.replace(userPath, '').replace(global.pathToApp + '/', '');
         var specID = path.dirname(cleanPath);
         var rawContents;
 

From 7130152cf4f026e77ccedc1c1b52199e383f8f23 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 25 Oct 2015 16:12:17 +0100
Subject: [PATCH 43/47] #190: align ejs sandboxing with new user path

---
 core/ejsWithHelpers.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/ejsWithHelpers.js b/core/ejsWithHelpers.js
index 99f2338..d76c425 100644
--- a/core/ejsWithHelpers.js
+++ b/core/ejsWithHelpers.js
@@ -97,6 +97,8 @@ var includeFiles = function(data, options){
 };
 
 ejs.render = function(template, data, options){
+    var sandboxPath = /^..\//.test(path.relative(global.pathToApp, global.userPath)) ? global.userPath : global.pathToApp;
+
     data = data || {};
     options = options || {};
 
@@ -113,7 +115,7 @@ ejs.render = function(template, data, options){
 
     if (global.opts.core.sandboxIncludes) {
         _.assign(options, {
-            sandbox: global.pathToApp
+            sandbox: sandboxPath
         });
     }
 

From 051ed50f268ac66e8ca1d611848ca9eeecce0fa7 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 25 Oct 2015 17:34:55 +0100
Subject: [PATCH 44/47] #190: fixes due to new user path

---
 app.js                  |  3 +++
 core/ejsWithHelpers.js  | 33 +++++++++++++++++++++++++++++----
 core/lib/configUtils.js |  2 +-
 core/lib/specUtils.js   |  2 +-
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/app.js b/app.js
index 9b33e1c..b9b05d4 100644
--- a/app.js
+++ b/app.js
@@ -40,6 +40,9 @@ global.commander = commander;
 var trackStats = require(path.join(global.pathToApp, 'core/trackStats'));
 var userPath = global.userPath = (require('./core/lib/getUserPath'))();
 
+// Legacy support
+app.set('user', userPath);
+
 console.log('Running user contents from', '`' + userPath + '`.');
 
 // We support `development` (default), `production` and `presentation` (for demos)
diff --git a/core/ejsWithHelpers.js b/core/ejsWithHelpers.js
index d76c425..bfb908d 100644
--- a/core/ejsWithHelpers.js
+++ b/core/ejsWithHelpers.js
@@ -2,6 +2,7 @@
 
 var ejs = require('ejs');
 var fs = require('fs');
+var nodeUtil = require('util');
 var glob = require('glob');
 var path = require('path');
 var _ = require('lodash');
@@ -20,7 +21,7 @@ var EJS_OPTS = ['cache', 'filename', 'sandbox', 'delimiter', 'scope', 'context',
  *
  * @memberof module:ejs-internal
  * @param {Object}  data data object
- * @param {Options} opts options object
+ * @param {Object} opts options object
  * @static
  */
 
@@ -32,8 +33,29 @@ var cpOptsInData = function (data, opts) {
     });
 };
 
+var isRestrictedReadPath = function(pathToRead, options) {
+  var isRescricted = true;
+  var i;
+
+  if (nodeUtil.isArray(options.sandbox)) {
+    for (i = 0; i < options.sandbox.length; i++) {
+      var sandboxPpath = options.sandbox[i];
+      var notRestricted = path.relative(sandboxPpath, pathToRead).substring(0, 2) !== '..';
+
+      if (notRestricted) {
+        isRescricted = false;
+        break;
+      }
+    }
+
+    return isRescricted;
+  } else {
+    return path.relative(options.sandbox, pathToRead).substring(0, 2) === '..';
+  }
+};
+
 var readFile = function (filePath, options) {
-    if (options.sandbox && path.relative(options.sandbox, filePath).substring(0, 2) === '..') {
+    if (options.sandbox && isRestrictedReadPath(filePath, options)) {
         throw new Error('reading files beyond sandbox is restricted, limit set to ' + options.sandbox);
     }
 
@@ -97,7 +119,10 @@ var includeFiles = function(data, options){
 };
 
 ejs.render = function(template, data, options){
-    var sandboxPath = /^..\//.test(path.relative(global.pathToApp, global.userPath)) ? global.userPath : global.pathToApp;
+    var sandboxPaths = [
+        global.pathToApp,
+        global.userPath
+    ];
 
     data = data || {};
     options = options || {};
@@ -115,7 +140,7 @@ ejs.render = function(template, data, options){
 
     if (global.opts.core.sandboxIncludes) {
         _.assign(options, {
-            sandbox: sandboxPath
+            sandbox: sandboxPaths
         });
     }
 
diff --git a/core/lib/configUtils.js b/core/lib/configUtils.js
index 8018df8..f050b98 100644
--- a/core/lib/configUtils.js
+++ b/core/lib/configUtils.js
@@ -180,7 +180,7 @@ module.exports.getContextOptions = function(refUrl, defaultOpts) {
     var infoOptionsKey = global.opts.core.common.infoFileOptions;
 
     // Extent context options object with info.json contents
-    contextOptions.specInfo = fs.readJsonSync(infoFilePath, {throws: false});
+    contextOptions.specInfo = fs.existsSync(infoFilePath) ? fs.readJsonSync(infoFilePath, {throws: false}) : undefined;
 
     // Override local options
     if (contextOptionsEnabled && contextOptions.specInfo && contextOptions.specInfo[infoOptionsKey]) {
diff --git a/core/lib/specUtils.js b/core/lib/specUtils.js
index 42440c1..0536244 100644
--- a/core/lib/specUtils.js
+++ b/core/lib/specUtils.js
@@ -117,7 +117,7 @@ module.exports.getFullPathToSpec = function(urlPath){
     // Including non-standard paths, outside default static route
     global.opts.core.common.includedDirs.forEach(function(item){
         if (cleanPath.split('/')[0] === item) {
-            specPath = specPath.replace('/' + global.opts.core.common.pathToUser + '/' + cleanPath, '/' + cleanPath);
+            specPath = specPath.replace(global.userPath, global.pathToApp + '/');
         }
     });
 

From 5391cb5f07bcf91e4a9a262ed4d393e07ccae939 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 25 Oct 2015 18:11:11 +0100
Subject: [PATCH 45/47] #190: fixes due to new user path

---
 assets/js/modules/sectionsParser.js | 2 +-
 core/file-tree/index.js             | 2 +-
 core/lib/getUserPath.js             | 2 +-
 core/middlewares/wrap.js            | 2 +-
 package.json                        | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/assets/js/modules/sectionsParser.js b/assets/js/modules/sectionsParser.js
index 05e65f1..50c9f81 100644
--- a/assets/js/modules/sectionsParser.js
+++ b/assets/js/modules/sectionsParser.js
@@ -211,7 +211,7 @@ function SourceGetSections() {
 /**
  * Filter list of HTML nodes
  *
- * @param {Array} elementsArr - nodelist to filter
+ * @param {Array} elementsArr - elements array to filter
  * @param {Function} [customElFilter] - Additional filter for element
  * @param {Boolean} [skipAttrFilters] - Set to true, if expect not filtered list of resources
  *
diff --git a/core/file-tree/index.js b/core/file-tree/index.js
index 081b32c..57b58ad 100644
--- a/core/file-tree/index.js
+++ b/core/file-tree/index.js
@@ -173,7 +173,7 @@ var getSpecMeta = module.exports.getSpecMeta = function(specDirOrPath){
     var relativeSpecPath = specLocation.relativeSpecPath;
 
     // Remove first slash for ID
-    page.id = relativeSpecPath === '/' ? '/' : relativeSpecPath.substring(1);
+    page.id = relativeSpecPath === '/' ? '/' : relativeSpecPath.replace(/^\//, '');
     page.url = relativeSpecPath;
 
     // If we have Spec, get additional meta
diff --git a/core/lib/getUserPath.js b/core/lib/getUserPath.js
index 6b095ce..e5157a6 100644
--- a/core/lib/getUserPath.js
+++ b/core/lib/getUserPath.js
@@ -12,7 +12,7 @@ module.exports = function(){
         pathToApp = sourcePathFromParent;
     }
 
-    var relativeUserPath = isNodeModule(pathToApp) ? '../../' : global.opts.core.common.pathToUser;
+    var relativeUserPath = isNodeModule(pathToApp) ? '../..' : global.opts.core.common.pathToUser;
 
     return path.join(pathToApp, relativeUserPath);
 };
\ No newline at end of file
diff --git a/core/middlewares/wrap.js b/core/middlewares/wrap.js
index efbc65f..4efb4b4 100644
--- a/core/middlewares/wrap.js
+++ b/core/middlewares/wrap.js
@@ -20,7 +20,7 @@ exports.process = function (req, res, next) {
 
     // Check if we're working with processed file
     if (req.specData && req.specData.renderedHtml) {
-        var specDir = specUtils.getFullPathToSpec(req.url);
+        var specDir = specUtils.getFullPathToSpec(req.path);
         var contextOptions = req.specData.contextOptions;
 
         // get spec content
diff --git a/package.json b/package.json
index df5532a..861adb7 100644
--- a/package.json
+++ b/package.json
@@ -51,7 +51,7 @@
   },
   "scripts": {
     "postinstall": "grunt update && node ./core/trackInstall",
-    "build": "node ./core/postInstall && grunt update",
+    "build": "grunt update",
     "lint": "grunt jshint",
     "start": "node app",
     "watch": "grunt watch-all",

From 4305bb086589025f97cac8bc3c157306c5ad4521 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 25 Oct 2015 18:21:30 +0100
Subject: [PATCH 46/47] optional dep for windows ci

---
 package.json | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 861adb7..5ec9fac 100644
--- a/package.json
+++ b/package.json
@@ -66,7 +66,6 @@
     "assert": "^1.3.0",
     "chai": "^3.2.0",
     "cross-spawn": "^0.4.0",
-    "grunt-casperjs": "^2.1.0",
     "grunt-contrib-cssmin": "^0.14.0",
     "grunt-contrib-htmlmin": "^0.5.0",
     "grunt-contrib-jshint": "^0.11.3",
@@ -77,5 +76,8 @@
     "mocha": "^2.0.1",
     "should": "^7.1.0",
     "supertest": "^1.1.0"
-  }
+  },
+  "optionalDependencies": {
+    "grunt-casperjs": "^2.1.0"
+  },
 }

From 42e3ac0af90c1573c34780504aa01201bf68e127 Mon Sep 17 00:00:00 2001
From: Robert Haritonov <r@rhr.me>
Date: Sun, 25 Oct 2015 18:23:25 +0100
Subject: [PATCH 47/47] fix json

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 5ec9fac..25a6178 100644
--- a/package.json
+++ b/package.json
@@ -79,5 +79,5 @@
   },
   "optionalDependencies": {
     "grunt-casperjs": "^2.1.0"
-  },
+  }
 }
