NoLog.cz/etherpad
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity

Merge pull request #3559 from raybellis/async-PR

Browse source 
With this commit, that closes #3540, we pay the first big slice of our technical
debt. In this line of work we streamlined the code base, reducing its size by
15-20% and making it more understandable at the same time.

The changes were audited and tested collaboratively and are deemed sufficiently
stable for being merged.

Known issues:
- plugin compatibility is still not perfect
- the error handling path needs to be improved

This is an important day for Etherpad: thanks, Ray!
This commit is contained in:
muxator 2019-03-07 02:04:29 +01:00 committed by GitHub
commit 4c45ac3cb1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 3984 additions and 5844 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

213
bin/checkAllPads.js
View file

@ -1,145 +1,94 @@
/* /*
This is a debug tool. It checks all revisions for data corruption * This is a debug tool. It checks all revisions for data corruption
*/ */
if(process.argv.length != 2) if (process.argv.length != 2) {
{
console.error("Use: node bin/checkAllPads.js"); console.error("Use: node bin/checkAllPads.js");
process.exit(1); process.exit(1);
} }
//initialize the variables // load and initialize NPM
var db, settings, padManager; let npm = require('../src/node_modules/npm');
var npm = require("../src/node_modules/npm"); npm.load({}, async function() {
var async = require("../src/node_modules/async");
var Changeset = require("../src/static/js/Changeset"); try {
// initialize the database
let settings = require('../src/node/utils/Settings');
let db = require('../src/node/db/DB');
await db.init();
async.series([ // load modules
//load npm let Changeset = require('../src/static/js/Changeset');
function(callback) { let padManager = require('../src/node/db/PadManager');
npm.load({}, callback);
},
//load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
//initialize the database // get all pads
db.init(callback); let res = await padManager.listAllPads();
},
//load pads
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.listAllPads(function(err, res)
{
padIds = res.padIDs;
callback(err);
});
},
function (callback)
{
async.forEach(padIds, function(padId, callback)
{
padManager.getPad(padId, function(err, pad) {
if (err) {
callback(err);
}
//check if the pad has a pool
if(pad.pool === undefined )
{
console.error("[" + pad.id + "] Missing attribute pool");
callback();
return;
}
//create an array with key kevisions for (let padId of res.padIDs) {
//key revisions always save the full pad atext
var head = pad.getHeadRevisionNumber();
var keyRevisions = [];
for(var i=0;i<head;i+=100)
{
keyRevisions.push(i);
}
//run trough all key revisions
async.forEachSeries(keyRevisions, function(keyRev, callback)
{
//create an array of revisions we need till the next keyRevision or the End
var revisionsNeeded = [];
for(var i=keyRev;i<=keyRev+100 && i<=head; i++)
{
revisionsNeeded.push(i);
}
//this array will hold all revision changesets
var revisions = [];
//run trough all needed revisions and get them from the database
async.forEach(revisionsNeeded, function(revNum, callback)
{
db.db.get("pad:"+pad.id+":revs:" + revNum, function(err, revision)
{
revisions[revNum] = revision;
callback(err);
});
}, function(err)
{
if(err)
{
callback(err);
return;
}
//check if the revision exists let pad = await padManager.getPad(padId);
if (revisions[keyRev] == null) {
console.error("[" + pad.id + "] Missing revision " + keyRev); // check if the pad has a pool
callback(); if (pad.pool === undefined) {
return; console.error("[" + pad.id + "] Missing attribute pool");
} continue;
}
//check if there is a atext in the keyRevisions
if(revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) // create an array with key kevisions
{ // key revisions always save the full pad atext
console.error("[" + pad.id + "] Missing atext in revision " + keyRev); let head = pad.getHeadRevisionNumber();
callback(); let keyRevisions = [];
return; for (let rev = 0; rev < head; rev += 100) {
} keyRevisions.push(rev);
}
var apool = pad.pool;
var atext = revisions[keyRev].meta.atext; // run through all key revisions
for (let keyRev of keyRevisions) {
for(var i=keyRev+1;i<=keyRev+100 && i<=head; i++)
{ // create an array of revisions we need till the next keyRevision or the End
try var revisionsNeeded = [];
{ for (let rev = keyRev ; rev <= keyRev + 100 && rev <= head; rev++) {
//console.log("[" + pad.id + "] check revision " + i); revisionsNeeded.push(rev);
var cs = revisions[i].changeset; }
atext = Changeset.applyToAText(cs, atext, apool);
} // this array will hold all revision changesets
catch(e) var revisions = [];
{
console.error("[" + pad.id + "] Bad changeset at revision " + i + " - " + e.message); // run through all needed revisions and get them from the database
callback(); for (let revNum of revisionsNeeded) {
return; let revision = await db.get("pad:" + pad.id + ":revs:" + revNum);
} revisions[revNum] = revision;
} }
callback(); // check if the revision exists
}); if (revisions[keyRev] == null) {
}, callback); console.error("[" + pad.id + "] Missing revision " + keyRev);
}); continue;
}, callback); }
}
], function (err) // check if there is a atext in the keyRevisions
{ if (revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) {
if(err) throw err; console.error("[" + pad.id + "] Missing atext in revision " + keyRev);
else continue;
{ }
console.log("finished");
process.exit(0); let apool = pad.pool;
let atext = revisions[keyRev].meta.atext;
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
try {
let cs = revisions[rev].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
} catch (e) {
console.error("[" + pad.id + "] Bad changeset at revision " + i + " - " + e.message);
}
}
}
console.log("finished");
process.exit(0);
}
} catch (err) {
console.trace(err);
process.exit(1);
} }
}); });

206
bin/checkPad.js
View file

@ -1,141 +1,95 @@
/* /*
This is a debug tool. It checks all revisions for data corruption * This is a debug tool. It checks all revisions for data corruption
*/ */
if(process.argv.length != 3) if (process.argv.length != 3) {
{
console.error("Use: node bin/checkPad.js $PADID"); console.error("Use: node bin/checkPad.js $PADID");
process.exit(1); process.exit(1);
} }
//get the padID
var padId = process.argv[2];
//initialize the variables // get the padID
var db, settings, padManager; const padId = process.argv[2];
var npm = require("../src/node_modules/npm");
var async = require("../src/node_modules/async");
var Changeset = require("ep_etherpad-lite/static/js/Changeset"); // load and initialize NPM;
let npm = require('../src/node_modules/npm');
npm.load({}, async function() {
async.series([ try {
//load npm // initialize database
function(callback) { let settings = require('../src/node/utils/Settings');
npm.load({}, function(er) { let db = require('../src/node/db/DB');
callback(er); await db.init();
})
},
//load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
//initialize the database // load modules
db.init(callback); let Changeset = require('ep_etherpad-lite/static/js/Changeset');
}, let padManager = require('../src/node/db/PadManager');
//get the pad
function (callback) let exists = await padManager.doesPadExists(padId);
{ if (!exists) {
padManager = require('../src/node/db/PadManager'); console.error("Pad does not exist");
process.exit(1);
padManager.doesPadExists(padId, function(err, exists) }
{
if(!exists) // get the pad
{ let pad = await padManager.getPad(padId);
console.error("Pad does not exist");
// create an array with key revisions
// key revisions always save the full pad atext
let head = pad.getHeadRevisionNumber();
let keyRevisions = [];
for (let rev = 0; rev < head; rev += 100) {
keyRevisions.push(rev);
}
// run through all key revisions
for (let keyRev of keyRevisions) {
// create an array of revisions we need till the next keyRevision or the End
let revisionsNeeded = [];
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
revisionsNeeded.push(rev);
}
// this array will hold all revision changesets
var revisions = [];
// run through all needed revisions and get them from the database
for (let revNum of revisionsNeeded) {
let revision = await db.get("pad:" + padId + ":revs:" + revNum);
revisions[revNum] = revision;
}
// check if the pad has a pool
if (pad.pool === undefined ) {
console.error("Attribute pool is missing");
process.exit(1); process.exit(1);
} }
padManager.getPad(padId, function(err, _pad) // check if there is an atext in the keyRevisions
{ if (revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) {
pad = _pad; console.error("No atext in key revision " + keyRev);
callback(err); continue;
});
});
},
function (callback)
{
//create an array with key revisions
//key revisions always save the full pad atext
var head = pad.getHeadRevisionNumber();
var keyRevisions = [];
for(var i=0;i<head;i+=100)
{
keyRevisions.push(i);
}
//run trough all key revisions
async.forEachSeries(keyRevisions, function(keyRev, callback)
{
//create an array of revisions we need till the next keyRevision or the End
var revisionsNeeded = [];
for(var i=keyRev;i<=keyRev+100 && i<=head; i++)
{
revisionsNeeded.push(i);
} }
//this array will hold all revision changesets let apool = pad.pool;
var revisions = []; let atext = revisions[keyRev].meta.atext;
//run trough all needed revisions and get them from the database for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
async.forEach(revisionsNeeded, function(revNum, callback) try {
{ // console.log("check revision " + rev);
db.db.get("pad:"+padId+":revs:" + revNum, function(err, revision) let cs = revisions[rev].changeset;
{ atext = Changeset.applyToAText(cs, atext, apool);
revisions[revNum] = revision; } catch(e) {
callback(err); console.error("Bad changeset at revision " + rev + " - " + e.message);
}); continue;
}, function(err)
{
if(err)
{
callback(err);
return;
} }
}
//check if the pad has a pool console.log("finished");
if(pad.pool === undefined ) process.exit(0);
{ }
console.error("Attribute pool is missing");
process.exit(1); } catch (e) {
} console.trace(e);
process.exit(1);
//check if there is an atext in the keyRevisions
if(revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined)
{
console.error("No atext in key revision " + keyRev);
callback();
return;
}
var apool = pad.pool;
var atext = revisions[keyRev].meta.atext;
for(var i=keyRev+1;i<=keyRev+100 && i<=head; i++)
{
try
{
//console.log("check revision " + i);
var cs = revisions[i].changeset;
atext = Changeset.applyToAText(cs, atext, apool);
}
catch(e)
{
console.error("Bad changeset at revision " + i + " - " + e.message);
callback();
return;
}
}
callback();
});
}, callback);
}
], function (err)
{
if(err) throw err;
else
{
console.log("finished");
process.exit(0);
} }
}); });

82
bin/deletePad.js
View file

@ -1,63 +1,41 @@
/* /*
A tool for deleting pads from the CLI, because sometimes a brick is required to fix a window. * A tool for deleting pads from the CLI, because sometimes a brick is required
*/ * to fix a window.
*/
if(process.argv.length != 3) if (process.argv.length != 3) {
{
console.error("Use: node deletePad.js $PADID"); console.error("Use: node deletePad.js $PADID");
process.exit(1); process.exit(1);
} }
//get the padID
var padId = process.argv[2];
var db, padManager, pad, settings; // get the padID
var neededDBValues = ["pad:"+padId]; let padId = process.argv[2];
var npm = require("../src/node_modules/npm"); let npm = require('../src/node_modules/npm');
var async = require("../src/node_modules/async");
async.series([ npm.load({}, async function(er) {
// load npm if (er) {
function(callback) { console.error("Could not load NPM: " + er)
npm.load({}, function(er) { process.exit(1);
if(er)
{
console.error("Could not load NPM: " + er)
process.exit(1);
}
else
{
callback();
}
})
},
// load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
callback();
},
// initialize the database
function (callback)
{
db.init(callback);
},
// delete the pad and its links
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.removePad(padId, function(err){
callback(err);
});
callback();
} }
], function (err)
{ try {
if(err) throw err; let settings = require('../src/node/utils/Settings');
else let db = require('../src/node/db/DB');
{ await db.init();
console.log("Finished deleting padId: "+padId);
process.exit(); padManager = require('../src/node/db/PadManager');
await padManager.removePad(padId);
console.log("Finished deleting padId: " + padId);
process.exit(0);
} catch (e) {
if (err.name === "apierror") {
console.error(e);
} else {
console.trace(e);
}
process.exit(1);
} }
}); });

162
bin/extractPadData.js
View file

@ -1,109 +1,75 @@
/* /*
This is a debug tool. It helps to extract all datas of a pad and move it from an productive environment and to a develop environment to reproduce bugs there. It outputs a dirtydb file * This is a debug tool. It helps to extract all datas of a pad and move it from
*/ * a productive environment and to a develop environment to reproduce bugs
* there. It outputs a dirtydb file
*/
if(process.argv.length != 3) if (process.argv.length != 3) {
{
console.error("Use: node extractPadData.js $PADID"); console.error("Use: node extractPadData.js $PADID");
process.exit(1); process.exit(1);
} }
//get the padID
var padId = process.argv[2];
var db, dirty, padManager, pad, settings; // get the padID
var neededDBValues = ["pad:"+padId]; let padId = process.argv[2];
var npm = require("../node_modules/ep_etherpad-lite/node_modules/npm"); let npm = require('../src/node_modules/npm');
var async = require("../node_modules/ep_etherpad-lite/node_modules/async");
async.series([ npm.load({}, async function(er) {
// load npm if (er) {
function(callback) { console.error("Could not load NPM: " + er)
npm.load({}, function(er) { process.exit(1);
if(er)
{
console.error("Could not load NPM: " + er)
process.exit(1);
}
else
{
callback();
}
})
},
// load modules
function(callback) {
settings = require('../node_modules/ep_etherpad-lite/node/utils/Settings');
db = require('../node_modules/ep_etherpad-lite/node/db/DB');
dirty = require("../node_modules/ep_etherpad-lite/node_modules/ueberDB/node_modules/dirty")(padId + ".db");
callback();
},
//initialize the database
function (callback)
{
db.init(callback);
},
//get the pad
function (callback)
{
padManager = require('../node_modules/ep_etherpad-lite/node/db/PadManager');
padManager.getPad(padId, function(err, _pad)
{
pad = _pad;
callback(err);
});
},
function (callback)
{
//add all authors
var authors = pad.getAllAuthors();
for(var i=0;i<authors.length;i++)
{
neededDBValues.push("globalAuthor:" + authors[i]);
}
//add all revisions
var revHead = pad.head;
for(var i=0;i<=revHead;i++)
{
neededDBValues.push("pad:"+padId+":revs:" + i);
}
//get all chat values
var chatHead = pad.chatHead;
for(var i=0;i<=chatHead;i++)
{
neededDBValues.push("pad:"+padId+":chat:" + i);
}
//get and set all values
async.forEach(neededDBValues, function(dbkey, callback)
{
db.db.db.wrappedDB.get(dbkey, function(err, dbvalue)
{
if(err) { callback(err); return}
if(dbvalue && typeof dbvalue != 'object'){
dbvalue=JSON.parse(dbvalue); // if it's not json then parse it as json
}
dirty.set(dbkey, dbvalue, callback);
});
}, callback);
} }
], function (err)
{ try {
if(err) throw err; // initialize database
else let settings = require('../src/node/utils/Settings');
{ let db = require('../src/node/db/DB');
console.log("finished"); await db.init();
process.exit();
// load extra modules
let dirtyDB = require('../src/node_modules/dirty');
let padManager = require('../src/node/db/PadManager');
let util = require('util');
// initialize output database
let dirty = dirtyDB(padId + '.db');
// Promise wrapped get and set function
let wrapped = db.db.db.wrappedDB;
let get = util.promisify(wrapped.get.bind(wrapped));
let set = util.promisify(dirty.set.bind(dirty));
// array in which required key values will be accumulated
let neededDBValues = ['pad:' + padId];
// get the actual pad object
let pad = await padManager.getPad(padId);
// add all authors
neededDBValues.push(...pad.getAllAuthors().map(author => 'globalAuthor:' + author));
// add all revisions
for (let rev = 0; rev <= pad.head; ++rev) {
neededDBValues.push('pad:' + padId + ':revs:' + rev);
}
// add all chat values
for (let chat = 0; chat <= pad.chatHead; ++chat) {
neededDBValues.push('pad:' + padId + ':chat:' + chat);
}
for (let dbkey of neededDBValues) {
let dbvalue = await get(dbkey);
if (dbvalue && typeof dbvalue !== 'object') {
dbvalue = JSON.parse(dbvalue);
}
await set(dbkey, dbvalue);
}
console.log('finished');
process.exit(0);
} catch (er) {
console.error(er);
process.exit(1);
} }
}); });
//get the pad object
//get all revisions of this pad
//get all authors related to this pad
//get the readonly link related to this pad
//get the chat entries related to this pad

144
bin/repairPad.js
View file

@ -1,106 +1,78 @@
/* /*
This is a repair tool. It extracts all datas of a pad, removes and inserts them again. * This is a repair tool. It extracts all datas of a pad, removes and inserts them again.
*/ */
console.warn("WARNING: This script must not be used while etherpad is running!"); console.warn("WARNING: This script must not be used while etherpad is running!");
if(process.argv.length != 3) if (process.argv.length != 3) {
{
console.error("Use: node bin/repairPad.js $PADID"); console.error("Use: node bin/repairPad.js $PADID");
process.exit(1); process.exit(1);
} }
//get the padID
// get the padID
var padId = process.argv[2]; var padId = process.argv[2];
var db, padManager, pad, settings; let npm = require("../src/node_modules/npm");
var neededDBValues = ["pad:"+padId]; npm.load({}, async function(er) {
if (er) {
console.error("Could not load NPM: " + er)
process.exit(1);
}
var npm = require("../src/node_modules/npm"); try {
var async = require("../src/node_modules/async"); // intialize database
let settings = require('../src/node/utils/Settings');
let db = require('../src/node/db/DB');
await db.init();
async.series([ // get the pad
// load npm let padManager = require('../src/node/db/PadManager');
function(callback) { let pad = await padManager.getPad(padId);
npm.load({}, function(er) {
if(er) // accumulate the required keys
{ let neededDBValues = ["pad:" + padId];
console.error("Could not load NPM: " + er)
process.exit(1); // add all authors
} neededDBValues.push(...pad.getAllAuthors().map(author => "globalAuthor:"));
else
{ // add all revisions
callback(); for (let rev = 0; rev <= pad.head; ++rev) {
} neededDBValues.push("pad:" + padId + ":revs:" + rev);
})
},
// load modules
function(callback) {
settings = require('../src/node/utils/Settings');
db = require('../src/node/db/DB');
callback();
},
//initialize the database
function (callback)
{
db.init(callback);
},
//get the pad
function (callback)
{
padManager = require('../src/node/db/PadManager');
padManager.getPad(padId, function(err, _pad)
{
pad = _pad;
callback(err);
});
},
function (callback)
{
//add all authors
var authors = pad.getAllAuthors();
for(var i=0;i<authors.length;i++)
{
neededDBValues.push("globalAuthor:" + authors[i]);
} }
//add all revisions // add all chat values
var revHead = pad.head; for (let chat = 0; chat <= pad.chatHead; ++chat) {
for(var i=0;i<=revHead;i++) neededDBValues.push("pad:" + padId + ":chat:" + chat);
{
neededDBValues.push("pad:"+padId+":revs:" + i);
} }
//get all chat values //
var chatHead = pad.chatHead; // NB: this script doesn't actually does what's documented
for(var i=0;i<=chatHead;i++) // since the `value` fields in the following `.forEach`
{ // block are just the array index numbers
neededDBValues.push("pad:"+padId+":chat:" + i); //
} // the script therefore craps out now before it can do
callback(); // any damage.
}, //
function (callback) { // See gitlab issue #3545
db = db.db; //
console.info("aborting [gitlab #3545]");
process.exit(1);
// now fetch and reinsert every key
neededDBValues.forEach(function(key, value) { neededDBValues.forEach(function(key, value) {
console.debug("Key: "+key+", value: "+value); console.log("Key: " + key+ ", value: " + value);
db.remove(key); db.remove(key);
db.set(key, value); db.set(key, value);
}); });
callback();
}
], function (err)
{
if(err) throw err;
else
{
console.info("finished"); console.info("finished");
process.exit(); process.exit(0);
} catch (er) {
if (er.name === "apierror") {
console.error(er);
} else {
console.trace(er);
}
} }
}); });
//get the pad object
//get all revisions of this pad
//get all authors related to this pad
//get the readonly link related to this pad
//get the chat entries related to this pad
//remove all keys from database and insert them again

1149
src/node/db/API.js
View file

File diff suppressed because it is too large Load diff

242
src/node/db/AuthorManager.js
View file

@ -18,211 +18,189 @@
* limitations under the License. * limitations under the License.
*/ */
var db = require("./DB");
var ERR = require("async-stacktrace");
var db = require("./DB").db;
var customError = require("../utils/customError"); var customError = require("../utils/customError");
var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString;
exports.getColorPalette = function(){ exports.getColorPalette = function() {
return ["#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1", "#ffa8a8", "#ffe699", "#cfff9e", "#99ffb3", "#a3ffff", "#99b3ff", "#cc99ff", "#ff99e5", "#e7b1b1", "#e9dcAf", "#cde9af", "#bfedcc", "#b1e7e7", "#c3cdee", "#d2b8ea", "#eec3e6", "#e9cece", "#e7e0ca", "#d3e5c7", "#bce1c5", "#c1e2e2", "#c1c9e2", "#cfc1e2", "#e0bdd9", "#baded3", "#a0f8eb", "#b1e7e0", "#c3c8e4", "#cec5e2", "#b1d5e7", "#cda8f0", "#f0f0a8", "#f2f2a6", "#f5a8eb", "#c5f9a9", "#ececbb", "#e7c4bc", "#daf0b2", "#b0a0fd", "#bce2e7", "#cce2bb", "#ec9afe", "#edabbd", "#aeaeea", "#c4e7b1", "#d722bb", "#f3a5e7", "#ffa8a8", "#d8c0c5", "#eaaedd", "#adc6eb", "#bedad1", "#dee9af", "#e9afc2", "#f8d2a0", "#b3b3e6"]; return [
"#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1",
"#ffa8a8", "#ffe699", "#cfff9e", "#99ffb3", "#a3ffff", "#99b3ff", "#cc99ff", "#ff99e5",
"#e7b1b1", "#e9dcAf", "#cde9af", "#bfedcc", "#b1e7e7", "#c3cdee", "#d2b8ea", "#eec3e6",
"#e9cece", "#e7e0ca", "#d3e5c7", "#bce1c5", "#c1e2e2", "#c1c9e2", "#cfc1e2", "#e0bdd9",
"#baded3", "#a0f8eb", "#b1e7e0", "#c3c8e4", "#cec5e2", "#b1d5e7", "#cda8f0", "#f0f0a8",
"#f2f2a6", "#f5a8eb", "#c5f9a9", "#ececbb", "#e7c4bc", "#daf0b2", "#b0a0fd", "#bce2e7",
"#cce2bb", "#ec9afe", "#edabbd", "#aeaeea", "#c4e7b1", "#d722bb", "#f3a5e7", "#ffa8a8",
"#d8c0c5", "#eaaedd", "#adc6eb", "#bedad1", "#dee9af", "#e9afc2", "#f8d2a0", "#b3b3e6"
];
}; };
/** /**
* Checks if the author exists * Checks if the author exists
*/ */
exports.doesAuthorExists = function (authorID, callback) exports.doesAuthorExist = async function(authorID)
{ {
//check if the database entry of this author exists let author = await db.get("globalAuthor:" + authorID);
db.get("globalAuthor:" + authorID, function (err, author)
{ return author !== null;
if(ERR(err, callback)) return;
callback(null, author != null);
});
} }
/* exported for backwards compatibility */
exports.doesAuthorExists = exports.doesAuthorExist;
/** /**
* Returns the AuthorID for a token. * Returns the AuthorID for a token.
* @param {String} token The token * @param {String} token The token
* @param {Function} callback callback (err, author)
*/ */
exports.getAuthor4Token = function (token, callback) exports.getAuthor4Token = async function(token)
{ {
mapAuthorWithDBKey("token2author", token, function(err, author) let author = await mapAuthorWithDBKey("token2author", token);
{
if(ERR(err, callback)) return; // return only the sub value authorID
//return only the sub value authorID return author ? author.authorID : author;
callback(null, author ? author.authorID : author);
});
} }
/** /**
* Returns the AuthorID for a mapper. * Returns the AuthorID for a mapper.
* @param {String} token The mapper * @param {String} token The mapper
* @param {String} name The name of the author (optional) * @param {String} name The name of the author (optional)
* @param {Function} callback callback (err, author)
*/ */
exports.createAuthorIfNotExistsFor = function (authorMapper, name, callback) exports.createAuthorIfNotExistsFor = async function(authorMapper, name)
{ {
mapAuthorWithDBKey("mapper2author", authorMapper, function(err, author) let author = await mapAuthorWithDBKey("mapper2author", authorMapper);
{
if(ERR(err, callback)) return;
//set the name of this author if (name) {
if(name) // set the name of this author
exports.setAuthorName(author.authorID, name); await exports.setAuthorName(author.authorID, name);
}
//return the authorID return author;
callback(null, author); };
});
}
/** /**
* Returns the AuthorID for a mapper. We can map using a mapperkey, * Returns the AuthorID for a mapper. We can map using a mapperkey,
* so far this is token2author and mapper2author * so far this is token2author and mapper2author
* @param {String} mapperkey The database key name for this mapper * @param {String} mapperkey The database key name for this mapper
* @param {String} mapper The mapper * @param {String} mapper The mapper
* @param {Function} callback callback (err, author)
*/ */
function mapAuthorWithDBKey (mapperkey, mapper, callback) async function mapAuthorWithDBKey (mapperkey, mapper)
{ {
//try to map to an author // try to map to an author
db.get(mapperkey + ":" + mapper, function (err, author) let author = await db.get(mapperkey + ":" + mapper);
{
if(ERR(err, callback)) return;
//there is no author with this mapper, so create one if (author === null) {
if(author == null) // there is no author with this mapper, so create one
{ let author = await exports.createAuthor(null);
exports.createAuthor(null, function(err, author)
{
if(ERR(err, callback)) return;
//create the token2author relation // create the token2author relation
db.set(mapperkey + ":" + mapper, author.authorID); await db.set(mapperkey + ":" + mapper, author.authorID);
//return the author // return the author
callback(null, author); return author;
}); }
return; // there is an author with this mapper
} // update the timestamp of this author
await db.setSub("globalAuthor:" + author, ["timestamp"], Date.now());
//there is a author with this mapper // return the author
//update the timestamp of this author return { authorID: author};
db.setSub("globalAuthor:" + author, ["timestamp"], Date.now());
//return the author
callback(null, {authorID: author});
});
} }
/** /**
* Internal function that creates the database entry for an author * Internal function that creates the database entry for an author
* @param {String} name The name of the author * @param {String} name The name of the author
*/ */
exports.createAuthor = function(name, callback) exports.createAuthor = function(name)
{ {
//create the new author name // create the new author name
var author = "a." + randomString(16); let author = "a." + randomString(16);
//create the globalAuthors db entry // create the globalAuthors db entry
var authorObj = {"colorId" : Math.floor(Math.random()*(exports.getColorPalette().length)), "name": name, "timestamp": Date.now()}; let authorObj = {
"colorId": Math.floor(Math.random() * (exports.getColorPalette().length)),
"name": name,
"timestamp": Date.now()
};
//set the global author db entry // set the global author db entry
// NB: no await, since we're not waiting for the DB set to finish
db.set("globalAuthor:" + author, authorObj); db.set("globalAuthor:" + author, authorObj);
callback(null, {authorID: author}); return { authorID: author };
} }
/** /**
* Returns the Author Obj of the author * Returns the Author Obj of the author
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {Function} callback callback(err, authorObj)
*/ */
exports.getAuthor = function (author, callback) exports.getAuthor = function(author)
{ {
db.get("globalAuthor:" + author, callback); // NB: result is already a Promise
return db.get("globalAuthor:" + author);
} }
/** /**
* Returns the color Id of the author * Returns the color Id of the author
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {Function} callback callback(err, colorId)
*/ */
exports.getAuthorColorId = function (author, callback) exports.getAuthorColorId = function(author)
{ {
db.getSub("globalAuthor:" + author, ["colorId"], callback); return db.getSub("globalAuthor:" + author, ["colorId"]);
} }
/** /**
* Sets the color Id of the author * Sets the color Id of the author
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {String} colorId The color id of the author * @param {String} colorId The color id of the author
* @param {Function} callback (optional)
*/ */
exports.setAuthorColorId = function (author, colorId, callback) exports.setAuthorColorId = function(author, colorId)
{ {
db.setSub("globalAuthor:" + author, ["colorId"], colorId, callback); return db.setSub("globalAuthor:" + author, ["colorId"], colorId);
} }
/** /**
* Returns the name of the author * Returns the name of the author
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {Function} callback callback(err, name)
*/ */
exports.getAuthorName = function (author, callback) exports.getAuthorName = function(author)
{ {
db.getSub("globalAuthor:" + author, ["name"], callback); return db.getSub("globalAuthor:" + author, ["name"]);
} }
/** /**
* Sets the name of the author * Sets the name of the author
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {String} name The name of the author * @param {String} name The name of the author
* @param {Function} callback (optional)
*/ */
exports.setAuthorName = function (author, name, callback) exports.setAuthorName = function(author, name)
{ {
db.setSub("globalAuthor:" + author, ["name"], name, callback); return db.setSub("globalAuthor:" + author, ["name"], name);
} }
/** /**
* Returns an array of all pads this author contributed to * Returns an array of all pads this author contributed to
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {Function} callback (optional)
*/ */
exports.listPadsOfAuthor = function (authorID, callback) exports.listPadsOfAuthor = async function(authorID)
{ {
/* There are two other places where this array is manipulated: /* There are two other places where this array is manipulated:
* (1) When the author is added to a pad, the author object is also updated * (1) When the author is added to a pad, the author object is also updated
* (2) When a pad is deleted, each author of that pad is also updated * (2) When a pad is deleted, each author of that pad is also updated
*/ */
//get the globalAuthor
db.get("globalAuthor:" + authorID, function(err, author)
{
if(ERR(err, callback)) return;
//author does not exists // get the globalAuthor
if(author == null) let author = await db.get("globalAuthor:" + authorID);
{
callback(new customError("authorID does not exist","apierror"))
return;
}
//everything is fine, return the pad IDs if (author === null) {
var pads = []; // author does not exist
if(author.padIDs != null) throw new customError("authorID does not exist", "apierror");
{ }
for (var padId in author.padIDs)
{ // everything is fine, return the pad IDs
pads.push(padId); let padIDs = Object.keys(author.padIDs || {});
}
} return { padIDs };
callback(null, {padIDs: pads});
});
} }
/** /**
@ -230,26 +208,27 @@ exports.listPadsOfAuthor = function (authorID, callback)
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {String} padID The id of the pad the author contributes to * @param {String} padID The id of the pad the author contributes to
*/ */
exports.addPad = function (authorID, padID) exports.addPad = async function(authorID, padID)
{ {
//get the entry // get the entry
db.get("globalAuthor:" + authorID, function(err, author) let author = await db.get("globalAuthor:" + authorID);
{
if(ERR(err)) return;
if(author == null) return;
//the entry doesn't exist so far, let's create it if (author === null) return;
if(author.padIDs == null)
{
author.padIDs = {};
}
//add the entry for this pad /*
author.padIDs[padID] = 1;// anything, because value is not used * ACHTUNG: padIDs can also be undefined, not just null, so it is not possible
* to perform a strict check here
*/
if (!author.padIDs) {
// the entry doesn't exist so far, let's create it
author.padIDs = {};
}
//save the new element back // add the entry for this pad
db.set("globalAuthor:" + authorID, author); author.padIDs[padID] = 1; // anything, because value is not used
});
// save the new element back
db.set("globalAuthor:" + authorID, author);
} }
/** /**
@ -257,18 +236,15 @@ exports.addPad = function (authorID, padID)
* @param {String} author The id of the author * @param {String} author The id of the author
* @param {String} padID The id of the pad the author contributes to * @param {String} padID The id of the pad the author contributes to
*/ */
exports.removePad = function (authorID, padID) exports.removePad = async function(authorID, padID)
{ {
db.get("globalAuthor:" + authorID, function (err, author) let author = await db.get("globalAuthor:" + authorID);
{
if(ERR(err)) return;
if(author == null) return;
if(author.padIDs != null) if (author === null) return;
{
//remove pad from author if (author.padIDs !== null) {
delete author.padIDs[padID]; // remove pad from author
db.set("globalAuthor:" + authorID, author); delete author.padIDs[padID];
} db.set("globalAuthor:" + authorID, author);
}); }
} }

60
src/node/db/DB.js
View file

@ -1,5 +1,5 @@
/** /**
* The DB Module provides a database initalized with the settings * The DB Module provides a database initalized with the settings
* provided by the settings module * provided by the settings module
*/ */
@ -22,9 +22,10 @@
var ueberDB = require("ueberdb2"); var ueberDB = require("ueberdb2");
var settings = require("../utils/Settings"); var settings = require("../utils/Settings");
var log4js = require('log4js'); var log4js = require('log4js');
const util = require("util");
//set database settings // set database settings
var db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB")); let db = new ueberDB.database(settings.dbType, settings.dbSettings, null, log4js.getLogger("ueberDB"));
/** /**
* The UeberDB Object that provides the database functions * The UeberDB Object that provides the database functions
@ -33,25 +34,40 @@ exports.db = null;
/** /**
* Initalizes the database with the settings provided by the settings module * Initalizes the database with the settings provided by the settings module
* @param {Function} callback * @param {Function} callback
*/ */
exports.init = function(callback) exports.init = function() {
{ // initalize the database async
//initalize the database async return new Promise((resolve, reject) => {
db.init(function(err) db.init(function(err) {
{ if (err) {
//there was an error while initializing the database, output it and stop // there was an error while initializing the database, output it and stop
if(err) console.error("ERROR: Problem while initalizing the database");
{ console.error(err.stack ? err.stack : err);
console.error("ERROR: Problem while initalizing the database"); process.exit(1);
console.error(err.stack ? err.stack : err); } else {
process.exit(1); // everything ok, set up Promise-based methods
} ['get', 'set', 'findKeys', 'getSub', 'setSub', 'remove', 'doShutdown'].forEach(fn => {
//everything ok exports[fn] = util.promisify(db[fn].bind(db));
else });
{
exports.db = db; // set up wrappers for get and getSub that can't return "undefined"
callback(null); let get = exports.get;
} exports.get = async function(key) {
let result = await get(key);
return (result === undefined) ? null : result;
};
let getSub = exports.getSub;
exports.getSub = async function(key, sub) {
let result = await getSub(key, sub);
return (result === undefined) ? null : result;
};
// exposed for those callers that need the underlying raw API
exports.db = db;
resolve();
}
});
}); });
} }

420
src/node/db/GroupManager.js
View file

@ -17,319 +17,167 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
var ERR = require("async-stacktrace");
var customError = require("../utils/customError"); var customError = require("../utils/customError");
var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString;
var db = require("./DB").db; var db = require("./DB");
var async = require("async");
var padManager = require("./PadManager"); var padManager = require("./PadManager");
var sessionManager = require("./SessionManager"); var sessionManager = require("./SessionManager");
exports.listAllGroups = function(callback) { exports.listAllGroups = async function()
db.get("groups", function (err, groups) {
if(ERR(err, callback)) return;
// there are no groups
if(groups == null) {
callback(null, {groupIDs: []});
return;
}
var groupIDs = [];
for ( var groupID in groups ) {
groupIDs.push(groupID);
}
callback(null, {groupIDs: groupIDs});
});
}
exports.deleteGroup = function(groupID, callback)
{ {
var group; let groups = await db.get("groups");
groups = groups || {};
async.series([ let groupIDs = Object.keys(groups);
//ensure group exists return { groupIDs };
function (callback)
{
//try to get the group entry
db.get("group:" + groupID, function (err, _group)
{
if(ERR(err, callback)) return;
//group does not exist
if(_group == null)
{
callback(new customError("groupID does not exist","apierror"));
return;
}
//group exists, everything is fine
group = _group;
callback();
});
},
//iterate trough all pads of this groups and delete them
function(callback)
{
//collect all padIDs in an array, that allows us to use async.forEach
var padIDs = [];
for(var i in group.pads)
{
padIDs.push(i);
}
//loop trough all pads and delete them
async.forEach(padIDs, function(padID, callback)
{
padManager.getPad(padID, function(err, pad)
{
if(ERR(err, callback)) return;
pad.remove(callback);
});
}, callback);
},
//iterate trough group2sessions and delete all sessions
function(callback)
{
//try to get the group entry
db.get("group2sessions:" + groupID, function (err, group2sessions)
{
if(ERR(err, callback)) return;
//skip if there is no group2sessions entry
if(group2sessions == null) {callback(); return}
//collect all sessions in an array, that allows us to use async.forEach
var sessions = [];
for(var i in group2sessions.sessionsIDs)
{
sessions.push(i);
}
//loop trough all sessions and delete them
async.forEach(sessions, function(session, callback)
{
sessionManager.deleteSession(session, callback);
}, callback);
});
},
//remove group and group2sessions entry
function(callback)
{
db.remove("group2sessions:" + groupID);
db.remove("group:" + groupID);
callback();
},
//unlist the group
function(callback)
{
exports.listAllGroups(function(err, groups) {
if(ERR(err, callback)) return;
groups = groups? groups.groupIDs : [];
// it's not listed
if(groups.indexOf(groupID) == -1) {
callback();
return;
}
groups.splice(groups.indexOf(groupID), 1);
// store empty groupe list
if(groups.length == 0) {
db.set("groups", {});
callback();
return;
}
// regenerate group list
var newGroups = {};
async.forEach(groups, function(group, cb) {
newGroups[group] = 1;
cb();
},function() {
db.set("groups", newGroups);
callback();
});
});
}
], function(err)
{
if(ERR(err, callback)) return;
callback();
});
}
exports.doesGroupExist = function(groupID, callback)
{
//try to get the group entry
db.get("group:" + groupID, function (err, group)
{
if(ERR(err, callback)) return;
callback(null, group != null);
});
} }
exports.createGroup = function(callback) exports.deleteGroup = async function(groupID)
{ {
//search for non existing groupID let group = await db.get("group:" + groupID);
var groupID = "g." + randomString(16);
// ensure group exists
//create the group if (group == null) {
db.set("group:" + groupID, {pads: {}}); // group does not exist
throw new customError("groupID does not exist", "apierror");
//list the group }
exports.listAllGroups(function(err, groups) {
if(ERR(err, callback)) return; // iterate through all pads of this group and delete them (in parallel)
groups = groups? groups.groupIDs : []; await Promise.all(Object.keys(group.pads).map(padID => {
return padManager.getPad(padID).then(pad => pad.remove());
groups.push(groupID); }));
// regenerate group list // iterate through group2sessions and delete all sessions
var newGroups = {}; let group2sessions = await db.get("group2sessions:" + groupID);
async.forEach(groups, function(group, cb) { let sessions = group2sessions ? group2sessions.sessionsIDs : {};
newGroups[group] = 1;
cb(); // loop through all sessions and delete them (in parallel)
},function() { await Promise.all(Object.keys(sessions).map(session => {
db.set("groups", newGroups); return sessionManager.deleteSession(session);
callback(null, {groupID: groupID}); }));
});
}); // remove group and group2sessions entry
} await db.remove("group2sessions:" + groupID);
await db.remove("group:" + groupID);
// unlist the group
let groups = await exports.listAllGroups();
groups = groups ? groups.groupIDs : [];
let index = groups.indexOf(groupID);
if (index === -1) {
// it's not listed
exports.createGroupIfNotExistsFor = function(groupMapper, callback)
{
//ensure mapper is optional
if(typeof groupMapper != "string")
{
callback(new customError("groupMapper is no string","apierror"));
return; return;
} }
//try to get a group for this mapper
db.get("mapper2group:"+groupMapper, function(err, groupID)
{
function createGroupForMapper(cb) {
exports.createGroup(function(err, responseObj)
{
if(ERR(err, cb)) return;
//create the mapper entry for this group
db.set("mapper2group:"+groupMapper, responseObj.groupID);
cb(null, responseObj);
});
}
if(ERR(err, callback)) return; // remove from the list
groups.splice(index, 1);
// regenerate group list
var newGroups = {};
groups.forEach(group => newGroups[group] = 1);
await db.set("groups", newGroups);
}
exports.doesGroupExist = async function(groupID)
{
// try to get the group entry
let group = await db.get("group:" + groupID);
return (group != null);
}
exports.createGroup = async function()
{
// search for non existing groupID
var groupID = "g." + randomString(16);
// create the group
await db.set("group:" + groupID, {pads: {}});
// list the group
let groups = await exports.listAllGroups();
groups = groups? groups.groupIDs : [];
groups.push(groupID);
// regenerate group list
var newGroups = {};
groups.forEach(group => newGroups[group] = 1);
await db.set("groups", newGroups);
return { groupID };
}
exports.createGroupIfNotExistsFor = async function(groupMapper)
{
// ensure mapper is optional
if (typeof groupMapper !== "string") {
throw new customError("groupMapper is not a string", "apierror");
}
// try to get a group for this mapper
let groupID = await db.get("mapper2group:" + groupMapper);
if (groupID) {
// there is a group for this mapper // there is a group for this mapper
if(groupID) { let exists = await exports.doesGroupExist(groupID);
exports.doesGroupExist(groupID, function(err, exists) {
if(ERR(err, callback)) return;
if(exists) return callback(null, {groupID: groupID});
// hah, the returned group doesn't exist, let's create one if (exists) return { groupID };
createGroupForMapper(callback) }
})
return; // hah, the returned group doesn't exist, let's create one
} let result = await exports.createGroup();
//there is no group for this mapper, let's create a group // create the mapper entry for this group
createGroupForMapper(callback) await db.set("mapper2group:" + groupMapper, result.groupID);
});
return result;
} }
exports.createGroupPad = function(groupID, padName, text, callback) exports.createGroupPad = async function(groupID, padName, text)
{ {
//create the padID // create the padID
var padID = groupID + "$" + padName; let padID = groupID + "$" + padName;
async.series([ // ensure group exists
//ensure group exists let groupExists = await exports.doesGroupExist(groupID);
function (callback)
{
exports.doesGroupExist(groupID, function(err, exists)
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist","apierror"));
return;
}
//group exists, everything is fine if (!groupExists) {
callback(); throw new customError("groupID does not exist", "apierror");
}); }
},
//ensure pad does not exists
function (callback)
{
padManager.doesPadExists(padID, function(err, exists)
{
if(ERR(err, callback)) return;
//pad exists already
if(exists == true)
{
callback(new customError("padName does already exist","apierror"));
return;
}
//pad does not exist, everything is fine // ensure pad doesn't exist already
callback(); let padExists = await padManager.doesPadExists(padID);
});
}, if (padExists) {
//create the pad // pad exists already
function (callback) throw new customError("padName does already exist", "apierror");
{ }
padManager.getPad(padID, text, function(err)
{ // create the pad
if(ERR(err, callback)) return; await padManager.getPad(padID, text);
callback();
}); //create an entry in the group for this pad
}, await db.setSub("group:" + groupID, ["pads", padID], 1);
//create an entry in the group for this pad
function (callback) return { padID };
{
db.setSub("group:" + groupID, ["pads", padID], 1);
callback();
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, {padID: padID});
});
} }
exports.listPads = function(groupID, callback) exports.listPads = async function(groupID)
{ {
exports.doesGroupExist(groupID, function(err, exists) let exists = await exports.doesGroupExist(groupID);
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist","apierror"));
return;
}
//group exists, let's get the pads // ensure the group exists
db.getSub("group:" + groupID, ["pads"], function(err, result) if (!exists) {
{ throw new customError("groupID does not exist", "apierror");
if(ERR(err, callback)) return; }
var pads = [];
for ( var padId in result ) { // group exists, let's get the pads
pads.push(padId); let result = await db.getSub("group:" + groupID, ["pads"]);
} let padIDs = Object.keys(result);
callback(null, {padIDs: pads});
}); return { padIDs };
});
} }

764
src/node/db/Pad.js
View file

@ -3,11 +3,9 @@
*/ */
var ERR = require("async-stacktrace");
var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var AttributePool = require("ep_etherpad-lite/static/js/AttributePool"); var AttributePool = require("ep_etherpad-lite/static/js/AttributePool");
var db = require("./DB").db; var db = require("./DB");
var async = require("async");
var settings = require('../utils/Settings'); var settings = require('../utils/Settings');
var authorManager = require("./AuthorManager"); var authorManager = require("./AuthorManager");
var padManager = require("./PadManager"); var padManager = require("./PadManager");
@ -19,7 +17,7 @@ var crypto = require("crypto");
var randomString = require("../utils/randomstring"); var randomString = require("../utils/randomstring");
var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks');
//serialization/deserialization attributes // serialization/deserialization attributes
var attributeBlackList = ["id"]; var attributeBlackList = ["id"];
var jsonableList = ["pool"]; var jsonableList = ["pool"];
@ -32,8 +30,7 @@ exports.cleanText = function (txt) {
}; };
var Pad = function Pad(id) { let Pad = function Pad(id) {
this.atext = Changeset.makeAText("\n"); this.atext = Changeset.makeAText("\n");
this.pool = new AttributePool(); this.pool = new AttributePool();
this.head = -1; this.head = -1;
@ -60,7 +57,7 @@ Pad.prototype.getSavedRevisionsNumber = function getSavedRevisionsNumber() {
Pad.prototype.getSavedRevisionsList = function getSavedRevisionsList() { Pad.prototype.getSavedRevisionsList = function getSavedRevisionsList() {
var savedRev = new Array(); var savedRev = new Array();
for(var rev in this.savedRevisions){ for (var rev in this.savedRevisions) {
savedRev.push(this.savedRevisions[rev].revNum); savedRev.push(this.savedRevisions[rev].revNum);
} }
savedRev.sort(function(a, b) { savedRev.sort(function(a, b) {
@ -74,8 +71,9 @@ Pad.prototype.getPublicStatus = function getPublicStatus() {
}; };
Pad.prototype.appendRevision = function appendRevision(aChangeset, author) { Pad.prototype.appendRevision = function appendRevision(aChangeset, author) {
if(!author) if (!author) {
author = ''; author = '';
}
var newAText = Changeset.applyToAText(aChangeset, this.atext, this.pool); var newAText = Changeset.applyToAText(aChangeset, this.atext, this.pool);
Changeset.copyAText(newAText, this.atext); Changeset.copyAText(newAText, this.atext);
@ -88,21 +86,22 @@ Pad.prototype.appendRevision = function appendRevision(aChangeset, author) {
newRevData.meta.author = author; newRevData.meta.author = author;
newRevData.meta.timestamp = Date.now(); newRevData.meta.timestamp = Date.now();
//ex. getNumForAuthor // ex. getNumForAuthor
if(author != '') if (author != '') {
this.pool.putAttrib(['author', author || '']); this.pool.putAttrib(['author', author || '']);
}
if(newRev % 100 == 0) if (newRev % 100 == 0) {
{
newRevData.meta.atext = this.atext; newRevData.meta.atext = this.atext;
} }
db.set("pad:"+this.id+":revs:"+newRev, newRevData); db.set("pad:" + this.id + ":revs:" + newRev, newRevData);
this.saveToDatabase(); this.saveToDatabase();
// set the author to pad // set the author to pad
if(author) if (author) {
authorManager.addPad(author, this.id); authorManager.addPad(author, this.id);
}
if (this.head == 0) { if (this.head == 0) {
hooks.callAll("padCreate", {'pad':this, 'author': author}); hooks.callAll("padCreate", {'pad':this, 'author': author});
@ -111,49 +110,47 @@ Pad.prototype.appendRevision = function appendRevision(aChangeset, author) {
} }
}; };
//save all attributes to the database // save all attributes to the database
Pad.prototype.saveToDatabase = function saveToDatabase(){ Pad.prototype.saveToDatabase = function saveToDatabase() {
var dbObject = {}; var dbObject = {};
for(var attr in this){ for (var attr in this) {
if(typeof this[attr] === "function") continue; if (typeof this[attr] === "function") continue;
if(attributeBlackList.indexOf(attr) !== -1) continue; if (attributeBlackList.indexOf(attr) !== -1) continue;
dbObject[attr] = this[attr]; dbObject[attr] = this[attr];
if(jsonableList.indexOf(attr) !== -1){ if (jsonableList.indexOf(attr) !== -1) {
dbObject[attr] = dbObject[attr].toJsonable(); dbObject[attr] = dbObject[attr].toJsonable();
} }
} }
db.set("pad:"+this.id, dbObject); db.set("pad:" + this.id, dbObject);
} }
// get time of last edit (changeset application) // get time of last edit (changeset application)
Pad.prototype.getLastEdit = function getLastEdit(callback){ Pad.prototype.getLastEdit = function getLastEdit() {
var revNum = this.getHeadRevisionNumber(); var revNum = this.getHeadRevisionNumber();
db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "timestamp"], callback); return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"]);
} }
Pad.prototype.getRevisionChangeset = function getRevisionChangeset(revNum, callback) { Pad.prototype.getRevisionChangeset = function getRevisionChangeset(revNum) {
db.getSub("pad:"+this.id+":revs:"+revNum, ["changeset"], callback); return db.getSub("pad:" + this.id + ":revs:" + revNum, ["changeset"]);
}; }
Pad.prototype.getRevisionAuthor = function getRevisionAuthor(revNum, callback) { Pad.prototype.getRevisionAuthor = function getRevisionAuthor(revNum) {
db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "author"], callback); return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "author"]);
}; }
Pad.prototype.getRevisionDate = function getRevisionDate(revNum, callback) { Pad.prototype.getRevisionDate = function getRevisionDate(revNum) {
db.getSub("pad:"+this.id+":revs:"+revNum, ["meta", "timestamp"], callback); return db.getSub("pad:" + this.id + ":revs:" + revNum, ["meta", "timestamp"]);
}; }
Pad.prototype.getAllAuthors = function getAllAuthors() { Pad.prototype.getAllAuthors = function getAllAuthors() {
var authors = []; var authors = [];
for(var key in this.pool.numToAttrib) for(var key in this.pool.numToAttrib) {
{ if (this.pool.numToAttrib[key][0] == "author" && this.pool.numToAttrib[key][1] != "") {
if(this.pool.numToAttrib[key][0] == "author" && this.pool.numToAttrib[key][1] != "")
{
authors.push(this.pool.numToAttrib[key][1]); authors.push(this.pool.numToAttrib[key][1]);
} }
} }
@ -161,120 +158,77 @@ Pad.prototype.getAllAuthors = function getAllAuthors() {
return authors; return authors;
}; };
Pad.prototype.getInternalRevisionAText = function getInternalRevisionAText(targetRev, callback) { Pad.prototype.getInternalRevisionAText = async function getInternalRevisionAText(targetRev) {
var _this = this; let keyRev = this.getKeyRevisionNumber(targetRev);
var keyRev = this.getKeyRevisionNumber(targetRev); // find out which changesets are needed
var atext; let neededChangesets = [];
var changesets = []; for (let curRev = keyRev; curRev < targetRev; ) {
neededChangesets.push(++curRev);
//find out which changesets are needed
var neededChangesets = [];
var curRev = keyRev;
while (curRev < targetRev)
{
curRev++;
neededChangesets.push(curRev);
} }
async.series([ // get all needed data out of the database
//get all needed data out of the database
function(callback)
{
async.parallel([
//get the atext of the key revision
function (callback)
{
db.getSub("pad:"+_this.id+":revs:"+keyRev, ["meta", "atext"], function(err, _atext)
{
if(ERR(err, callback)) return;
try {
atext = Changeset.cloneAText(_atext);
} catch (e) {
return callback(e);
}
callback(); // start to get the atext of the key revision
}); let p_atext = db.getSub("pad:" + this.id + ":revs:" + keyRev, ["meta", "atext"]);
},
//get all needed changesets
function (callback)
{
async.forEach(neededChangesets, function(item, callback)
{
_this.getRevisionChangeset(item, function(err, changeset)
{
if(ERR(err, callback)) return;
changesets[item] = changeset;
callback();
});
}, callback);
}
], callback);
},
//apply all changesets to the key changeset
function(callback)
{
var apool = _this.apool();
var curRev = keyRev;
while (curRev < targetRev) // get all needed changesets
{ let changesets = [];
curRev++; await Promise.all(neededChangesets.map(item => {
var cs = changesets[curRev]; return this.getRevisionChangeset(item).then(changeset => {
try{ changesets[item] = changeset;
atext = Changeset.applyToAText(cs, atext, apool);
}catch(e) {
return callback(e)
}
}
callback(null);
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, atext);
});
};
Pad.prototype.getRevision = function getRevisionChangeset(revNum, callback) {
db.get("pad:"+this.id+":revs:"+revNum, callback);
};
Pad.prototype.getAllAuthorColors = function getAllAuthorColors(callback){
var authors = this.getAllAuthors();
var returnTable = {};
var colorPalette = authorManager.getColorPalette();
async.forEach(authors, function(author, callback){
authorManager.getAuthorColorId(author, function(err, colorId){
if(err){
return callback(err);
}
//colorId might be a hex color or an number out of the palette
returnTable[author]=colorPalette[colorId] || colorId;
callback();
}); });
}, function(err){ }));
callback(err, returnTable);
}); // we should have the atext by now
}; let atext = await p_atext;
atext = Changeset.cloneAText(atext);
// apply all changesets to the key changeset
let apool = this.apool();
for (let curRev = keyRev; curRev < targetRev; ) {
let cs = changesets[++curRev];
atext = Changeset.applyToAText(cs, atext, apool);
}
return atext;
}
Pad.prototype.getRevision = function getRevisionChangeset(revNum) {
return db.get("pad:" + this.id + ":revs:" + revNum);
}
Pad.prototype.getAllAuthorColors = async function getAllAuthorColors() {
let authors = this.getAllAuthors();
let returnTable = {};
let colorPalette = authorManager.getColorPalette();
await Promise.all(authors.map(author => {
return authorManager.getAuthorColorId(author).then(colorId => {
// colorId might be a hex color or an number out of the palette
returnTable[author] = colorPalette[colorId] || colorId;
});
}));
return returnTable;
}
Pad.prototype.getValidRevisionRange = function getValidRevisionRange(startRev, endRev) { Pad.prototype.getValidRevisionRange = function getValidRevisionRange(startRev, endRev) {
startRev = parseInt(startRev, 10); startRev = parseInt(startRev, 10);
var head = this.getHeadRevisionNumber(); var head = this.getHeadRevisionNumber();
endRev = endRev ? parseInt(endRev, 10) : head; endRev = endRev ? parseInt(endRev, 10) : head;
if(isNaN(startRev) || startRev < 0 || startRev > head) {
if (isNaN(startRev) || startRev < 0 || startRev > head) {
startRev = null; startRev = null;
} }
if(isNaN(endRev) || endRev < startRev) {
if (isNaN(endRev) || endRev < startRev) {
endRev = null; endRev = null;
} else if(endRev > head) { } else if (endRev > head) {
endRev = head; endRev = head;
} }
if(startRev !== null && endRev !== null) {
if (startRev !== null && endRev !== null) {
return { startRev: startRev , endRev: endRev } return { startRev: startRev , endRev: endRev }
} }
return null; return null;
@ -289,12 +243,12 @@ Pad.prototype.text = function text() {
}; };
Pad.prototype.setText = function setText(newText) { Pad.prototype.setText = function setText(newText) {
//clean the new text // clean the new text
newText = exports.cleanText(newText); newText = exports.cleanText(newText);
var oldText = this.text(); var oldText = this.text();
//create the changeset // create the changeset
// We want to ensure the pad still ends with a \n, but otherwise keep // We want to ensure the pad still ends with a \n, but otherwise keep
// getText() and setText() consistent. // getText() and setText() consistent.
var changeset; var changeset;
@ -304,155 +258,105 @@ Pad.prototype.setText = function setText(newText) {
changeset = Changeset.makeSplice(oldText, 0, oldText.length-1, newText); changeset = Changeset.makeSplice(oldText, 0, oldText.length-1, newText);
} }
//append the changeset // append the changeset
this.appendRevision(changeset); this.appendRevision(changeset);
}; };
Pad.prototype.appendText = function appendText(newText) { Pad.prototype.appendText = function appendText(newText) {
//clean the new text // clean the new text
newText = exports.cleanText(newText); newText = exports.cleanText(newText);
var oldText = this.text(); var oldText = this.text();
//create the changeset // create the changeset
var changeset = Changeset.makeSplice(oldText, oldText.length, 0, newText); var changeset = Changeset.makeSplice(oldText, oldText.length, 0, newText);
//append the changeset // append the changeset
this.appendRevision(changeset); this.appendRevision(changeset);
}; };
Pad.prototype.appendChatMessage = function appendChatMessage(text, userId, time) { Pad.prototype.appendChatMessage = function appendChatMessage(text, userId, time) {
this.chatHead++; this.chatHead++;
//save the chat entry in the database // save the chat entry in the database
db.set("pad:"+this.id+":chat:"+this.chatHead, {"text": text, "userId": userId, "time": time}); db.set("pad:" + this.id + ":chat:" + this.chatHead, { "text": text, "userId": userId, "time": time });
this.saveToDatabase(); this.saveToDatabase();
}; };
Pad.prototype.getChatMessage = function getChatMessage(entryNum, callback) { Pad.prototype.getChatMessage = async function getChatMessage(entryNum) {
var _this = this; // get the chat entry
var entry; let entry = await db.get("pad:" + this.id + ":chat:" + entryNum);
async.series([ // get the authorName if the entry exists
//get the chat entry if (entry != null) {
function(callback) entry.userName = await authorManager.getAuthorName(entry.userId);
{
db.get("pad:"+_this.id+":chat:"+entryNum, function(err, _entry)
{
if(ERR(err, callback)) return;
entry = _entry;
callback();
});
},
//add the authorName
function(callback)
{
//this chat message doesn't exist, return null
if(entry == null)
{
callback();
return;
}
//get the authorName
authorManager.getAuthorName(entry.userId, function(err, authorName)
{
if(ERR(err, callback)) return;
entry.userName = authorName;
callback();
});
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, entry);
});
};
Pad.prototype.getChatMessages = function getChatMessages(start, end, callback) {
//collect the numbers of chat entries and in which order we need them
var neededEntries = [];
var order = 0;
for(var i=start;i<=end; i++)
{
neededEntries.push({entryNum:i, order: order});
order++;
} }
var _this = this; return entry;
//get all entries out of the database
var entries = [];
async.forEach(neededEntries, function(entryObject, callback)
{
_this.getChatMessage(entryObject.entryNum, function(err, entry)
{
if(ERR(err, callback)) return;
entries[entryObject.order] = entry;
callback();
});
}, function(err)
{
if(ERR(err, callback)) return;
//sort out broken chat entries
//it looks like in happend in the past that the chat head was
//incremented, but the chat message wasn't added
var cleanedEntries = [];
for(var i=0;i<entries.length;i++)
{
if(entries[i]!=null)
cleanedEntries.push(entries[i]);
else
console.warn("WARNING: Found broken chat entry in pad " + _this.id);
}
callback(null, cleanedEntries);
});
}; };
Pad.prototype.init = function init(text, callback) { Pad.prototype.getChatMessages = async function getChatMessages(start, end) {
var _this = this;
//replace text with default text if text isn't set // collect the numbers of chat entries and in which order we need them
if(text == null) let neededEntries = [];
{ for (let order = 0, entryNum = start; entryNum <= end; ++order, ++entryNum) {
neededEntries.push({ entryNum, order });
}
// get all entries out of the database
let entries = [];
await Promise.all(neededEntries.map(entryObject => {
return this.getChatMessage(entryObject.entryNum).then(entry => {
entries[entryObject.order] = entry;
});
}));
// sort out broken chat entries
// it looks like in happened in the past that the chat head was
// incremented, but the chat message wasn't added
let cleanedEntries = entries.filter(entry => {
let pass = (entry != null);
if (!pass) {
console.warn("WARNING: Found broken chat entry in pad " + this.id);
}
return pass;
});
return cleanedEntries;
}
Pad.prototype.init = async function init(text) {
// replace text with default text if text isn't set
if (text == null) {
text = settings.defaultPadText; text = settings.defaultPadText;
} }
//try to load the pad // try to load the pad
db.get("pad:"+this.id, function(err, value) let value = await db.get("pad:" + this.id);
{
if(ERR(err, callback)) return;
//if this pad exists, load it // if this pad exists, load it
if(value != null) if (value != null) {
{ // copy all attr. To a transfrom via fromJsonable if necassary
//copy all attr. To a transfrom via fromJsonable if necassary for (var attr in value) {
for(var attr in value){ if (jsonableList.indexOf(attr) !== -1) {
if(jsonableList.indexOf(attr) !== -1){ this[attr] = this[attr].fromJsonable(value[attr]);
_this[attr] = _this[attr].fromJsonable(value[attr]); } else {
} else { this[attr] = value[attr];
_this[attr] = value[attr];
}
} }
} }
//this pad doesn't exist, so create it } else {
else // this pad doesn't exist, so create it
{ let firstChangeset = Changeset.makeSplice("\n", 0, 0, exports.cleanText(text));
var firstChangeset = Changeset.makeSplice("\n", 0, 0, exports.cleanText(text));
_this.appendRevision(firstChangeset, ''); this.appendRevision(firstChangeset, '');
} }
hooks.callAll("padLoad", {'pad':_this}); hooks.callAll("padLoad", { 'pad': this });
callback(null); }
});
};
Pad.prototype.copy = function copy(destinationID, force, callback) { Pad.prototype.copy = async function copy(destinationID, force) {
var sourceID = this.id;
var _this = this; let sourceID = this.id;
var destGroupID;
// allow force to be a string // allow force to be a string
if (typeof force === "string") { if (typeof force === "string") {
@ -467,247 +371,139 @@ Pad.prototype.copy = function copy(destinationID, force, callback) {
// padMessageHandler.kickSessionsFromPad(sourceID); // padMessageHandler.kickSessionsFromPad(sourceID);
// flush the source pad: // flush the source pad:
_this.saveToDatabase(); this.saveToDatabase();
async.series([ // if it's a group pad, let's make sure the group exists.
// if it's a group pad, let's make sure the group exists. let destGroupID;
function(callback) if (destinationID.indexOf("$") >= 0) {
{
if (destinationID.indexOf("$") === -1)
{
callback();
return;
}
destGroupID = destinationID.split("$")[0] destGroupID = destinationID.split("$")[0]
groupManager.doesGroupExist(destGroupID, function (err, exists) let groupExists = await groupManager.doesGroupExist(destGroupID);
{
if(ERR(err, callback)) return;
//group does not exist // group does not exist
if(exists == false) if (!groupExists) {
{ throw new customError("groupID does not exist for destinationID", "apierror");
callback(new customError("groupID does not exist for destinationID","apierror"));
return;
}
//everything is fine, continue
callback();
});
},
// if the pad exists, we should abort, unless forced.
function(callback)
{
padManager.doesPadExists(destinationID, function (err, exists)
{
if(ERR(err, callback)) return;
/*
* this is the negation of a truthy comparison. Has been left in this
* wonky state to keep the old (possibly buggy) behaviour
*/
if (!(exists == true))
{
callback();
return;
}
if (!force)
{
console.error("erroring out without force");
callback(new customError("destinationID already exists","apierror"));
console.error("erroring out without force - after");
return;
}
// exists and forcing
padManager.getPad(destinationID, function(err, pad) {
if (ERR(err, callback)) return;
pad.remove(callback);
});
});
},
// copy the 'pad' entry
function(callback)
{
db.get("pad:"+sourceID, function(err, pad) {
db.set("pad:"+destinationID, pad);
});
callback();
},
//copy all relations
function(callback)
{
async.parallel([
//copy all chat messages
function(callback)
{
var chatHead = _this.chatHead;
for(var i=0;i<=chatHead;i++)
{
db.get("pad:"+sourceID+":chat:"+i, function (err, chat) {
if (ERR(err, callback)) return;
db.set("pad:"+destinationID+":chat:"+i, chat);
});
}
callback();
},
//copy all revisions
function(callback)
{
var revHead = _this.head;
for(var i=0;i<=revHead;i++)
{
db.get("pad:"+sourceID+":revs:"+i, function (err, rev) {
if (ERR(err, callback)) return;
db.set("pad:"+destinationID+":revs:"+i, rev);
});
}
callback();
},
//add the new pad to all authors who contributed to the old one
function(callback)
{
var authorIDs = _this.getAllAuthors();
authorIDs.forEach(function (authorID)
{
authorManager.addPad(authorID, destinationID);
});
callback();
},
// parallel
], callback);
},
function(callback) {
// Group pad? Add it to the group's list
if(destGroupID) db.setSub("group:" + destGroupID, ["pads", destinationID], 1);
// Initialize the new pad (will update the listAllPads cache)
setTimeout(function(){
padManager.getPad(destinationID, null, callback) // this runs too early.
},10);
},
// let the plugins know the pad was copied
function(callback) {
hooks.callAll('padCopy', { 'originalPad': _this, 'destinationID': destinationID });
callback();
} }
// series }
], function(err)
{ // if the pad exists, we should abort, unless forced.
if(ERR(err, callback)) return; let exists = await padManager.doesPadExist(destinationID);
callback(null, {padID: destinationID});
if (exists) {
if (!force) {
console.error("erroring out without force");
throw new customError("destinationID already exists", "apierror");
return;
}
// exists and forcing
let pad = await padManager.getPad(destinationID);
await pad.remove(callback);
}
// copy the 'pad' entry
let pad = await db.get("pad:" + sourceID);
db.set("pad:" + destinationID, pad);
// copy all relations in parallel
let promises = [];
// copy all chat messages
let chatHead = this.chatHead;
for (let i = 0; i <= chatHead; ++i) {
let p = db.get("pad:" + sourceID + ":chat:" + i).then(chat => {
return db.set("pad:" + destinationID + ":chat:" + i, chat);
});
promises.push(p);
}
// copy all revisions
let revHead = this.head;
for (let i = 0; i <= revHead; ++i) {
let p = db.get("pad:" + sourceID + ":revs:" + i).then(rev => {
return db.set("pad:" + destinationID + ":revs:" + i, rev);
});
promises.push(p);
}
// add the new pad to all authors who contributed to the old one
this.getAllAuthors().forEach(authorID => {
authorManager.addPad(authorID, destinationID);
}); });
};
Pad.prototype.remove = function remove(callback) { // wait for the above to complete
await Promise.all(promises);
// Group pad? Add it to the group's list
if (destGroupID) {
await db.setSub("group:" + destGroupID, ["pads", destinationID], 1);
}
// delay still necessary?
await new Promise(resolve => setTimeout(resolve, 10));
// Initialize the new pad (will update the listAllPads cache)
await padManager.getPad(destinationID, null); // this runs too early.
// let the plugins know the pad was copied
hooks.callAll('padCopy', { 'originalPad': this, 'destinationID': destinationID });
return { padID: destinationID };
}
Pad.prototype.remove = async function remove() {
var padID = this.id; var padID = this.id;
var _this = this;
//kick everyone from this pad // kick everyone from this pad
padMessageHandler.kickSessionsFromPad(padID); padMessageHandler.kickSessionsFromPad(padID);
async.series([ // delete all relations - the original code used async.parallel but
//delete all relations // none of the operations except getting the group depended on callbacks
function(callback) // so the database operations here are just started and then left to
{ // run to completion
async.parallel([
//is it a group pad? -> delete the entry of this pad in the group // is it a group pad? -> delete the entry of this pad in the group
function(callback) if (padID.indexOf("$") >= 0) {
{
if(padID.indexOf("$") === -1)
{
// it isn't a group pad, nothing to do here
callback();
return;
}
// it is a group pad // it is a group pad
var groupID = padID.substring(0,padID.indexOf("$")); let groupID = padID.substring(0, padID.indexOf("$"));
let group = await db.get("group:" + groupID);
db.get("group:" + groupID, function (err, group) // remove the pad entry
{ delete group.pads[padID];
if(ERR(err, callback)) return;
//remove the pad entry // set the new value
delete group.pads[padID]; db.set("group:" + groupID, group);
}
//set the new value // remove the readonly entries
db.set("group:" + groupID, group); let readonlyID = readOnlyManager.getReadOnlyId(padID);
callback(); db.remove("pad2readonly:" + padID);
}); db.remove("readonly2pad:" + readonlyID);
},
//remove the readonly entries
function(callback)
{
readOnlyManager.getReadOnlyId(padID, function(err, readonlyID)
{
if(ERR(err, callback)) return;
db.remove("pad2readonly:" + padID); // delete all chat messages
db.remove("readonly2pad:" + readonlyID); for (let i = 0, n = this.chatHead; i <= n; ++i) {
db.remove("pad:" + padID + ":chat:" + i);
}
callback(); // delete all revisions
}); for (let i = 0, n = this.head; i <= n; ++i) {
}, db.remove("pad:" + padID + ":revs:" + i);
//delete all chat messages }
function(callback)
{
var chatHead = _this.chatHead;
for(var i=0;i<=chatHead;i++) // remove pad from all authors who contributed
{ this.getAllAuthors().forEach(authorID => {
db.remove("pad:"+padID+":chat:"+i); authorManager.removePad(authorID, padID);
}
callback();
},
//delete all revisions
function(callback)
{
var revHead = _this.head;
for(var i=0;i<=revHead;i++)
{
db.remove("pad:"+padID+":revs:"+i);
}
callback();
},
//remove pad from all authors who contributed
function(callback)
{
var authorIDs = _this.getAllAuthors();
authorIDs.forEach(function (authorID)
{
authorManager.removePad(authorID, padID);
});
callback();
}
], callback);
},
//delete the pad entry and delete pad from padManager
function(callback)
{
padManager.removePad(padID);
hooks.callAll("padRemove", {'padID':padID});
callback();
}
], function(err)
{
if(ERR(err, callback)) return;
callback();
}); });
};
//set in db // delete the pad entry and delete pad from padManager
padManager.removePad(padID);
hooks.callAll("padRemove", { padID });
}
// set in db
Pad.prototype.setPublicStatus = function setPublicStatus(publicStatus) { Pad.prototype.setPublicStatus = function setPublicStatus(publicStatus) {
this.publicStatus = publicStatus; this.publicStatus = publicStatus;
this.saveToDatabase(); this.saveToDatabase();
@ -727,14 +523,14 @@ Pad.prototype.isPasswordProtected = function isPasswordProtected() {
}; };
Pad.prototype.addSavedRevision = function addSavedRevision(revNum, savedById, label) { Pad.prototype.addSavedRevision = function addSavedRevision(revNum, savedById, label) {
//if this revision is already saved, return silently // if this revision is already saved, return silently
for(var i in this.savedRevisions){ for (var i in this.savedRevisions) {
if(this.savedRevisions[i] && this.savedRevisions[i].revNum === revNum){ if (this.savedRevisions[i] && this.savedRevisions[i].revNum === revNum) {
return; return;
} }
} }
//build the saved revision object // build the saved revision object
var savedRevision = {}; var savedRevision = {};
savedRevision.revNum = revNum; savedRevision.revNum = revNum;
savedRevision.savedById = savedById; savedRevision.savedById = savedById;
@ -742,7 +538,7 @@ Pad.prototype.addSavedRevision = function addSavedRevision(revNum, savedById, la
savedRevision.timestamp = Date.now(); savedRevision.timestamp = Date.now();
savedRevision.id = randomString(10); savedRevision.id = randomString(10);
//save this new saved revision // save this new saved revision
this.savedRevisions.push(savedRevision); this.savedRevisions.push(savedRevision);
this.saveToDatabase(); this.saveToDatabase();
}; };
@ -753,19 +549,17 @@ Pad.prototype.getSavedRevisions = function getSavedRevisions() {
/* Crypto helper methods */ /* Crypto helper methods */
function hash(password, salt) function hash(password, salt) {
{
var shasum = crypto.createHash('sha512'); var shasum = crypto.createHash('sha512');
shasum.update(password + salt); shasum.update(password + salt);
return shasum.digest("hex") + "$" + salt; return shasum.digest("hex") + "$" + salt;
} }
function generateSalt() function generateSalt() {
{
return randomString(86); return randomString(86);
} }
function compare(hashStr, password) function compare(hashStr, password) {
{
return hash(password, hashStr.split("$")[1]) === hashStr; return hash(password, hashStr.split("$")[1]) === hashStr;
} }

280
src/node/db/PadManager.js
View file

@ -18,12 +18,11 @@
* limitations under the License. * limitations under the License.
*/ */
var ERR = require("async-stacktrace");
var customError = require("../utils/customError"); var customError = require("../utils/customError");
var Pad = require("../db/Pad").Pad; var Pad = require("../db/Pad").Pad;
var db = require("./DB").db; var db = require("./DB");
/** /**
* A cache of all loaded Pads. * A cache of all loaded Pads.
* *
* Provides "get" and "set" functions, * Provides "get" and "set" functions,
@ -35,12 +34,11 @@ var db = require("./DB").db;
* that's defined somewhere more sensible. * that's defined somewhere more sensible.
*/ */
var globalPads = { var globalPads = {
get: function (name) { return this[':'+name]; }, get: function(name) { return this[':'+name]; },
set: function (name, value) set: function(name, value) {
{
this[':'+name] = value; this[':'+name] = value;
}, },
remove: function (name) { remove: function(name) {
delete this[':'+name]; delete this[':'+name];
} }
}; };
@ -50,183 +48,151 @@ var globalPads = {
* *
* Updated without db access as new pads are created/old ones removed. * Updated without db access as new pads are created/old ones removed.
*/ */
var padList = { let padList = {
list: [], list: [],
sorted : false, sorted : false,
initiated: false, initiated: false,
init: function(cb) init: async function() {
{ let dbData = await db.findKeys("pad:*", "*:*:*");
db.findKeys("pad:*", "*:*:*", function(err, dbData)
{ if (dbData != null) {
if(ERR(err, cb)) return; this.initiated = true;
if(dbData != null){
padList.initiated = true for (let val of dbData) {
dbData.forEach(function(val){ this.addPad(val.replace(/pad:/,""), false);
padList.addPad(val.replace(/pad:/,""),false);
});
cb && cb()
} }
}); }
return this; return this;
}, },
load: function(cb) { load: async function() {
if(this.initiated) cb && cb() if (!this.initiated) {
else this.init(cb) return this.init();
}
return this;
}, },
/** /**
* Returns all pads in alphabetical order as array. * Returns all pads in alphabetical order as array.
*/ */
getPads: function(cb){ getPads: async function() {
this.load(function() { await this.load();
if(!padList.sorted){
padList.list = padList.list.sort(); if (!this.sorted) {
padList.sorted = true; this.list.sort();
} this.sorted = true;
cb && cb(padList.list); }
})
return this.list;
}, },
addPad: function(name) addPad: function(name) {
{ if (!this.initiated) return;
if(!this.initiated) return;
if(this.list.indexOf(name) == -1){ if (this.list.indexOf(name) == -1) {
this.list.push(name); this.list.push(name);
this.sorted=false; this.sorted = false;
} }
}, },
removePad: function(name) removePad: function(name) {
{ if (!this.initiated) return;
if(!this.initiated) return;
var index = this.list.indexOf(name); var index = this.list.indexOf(name);
if(index>-1){
this.list.splice(index,1); if (index > -1) {
this.sorted=false; this.list.splice(index, 1);
this.sorted = false;
} }
} }
}; };
//initialises the allknowing data structure
// initialises the all-knowing data structure
/**
* Returns a Pad Object with the callback
* @param id A String with the id of the pad
* @param {Function} callback
*/
exports.getPad = async function(id, text)
{
// check if this is a valid padId
if (!exports.isValidPadId(id)) {
throw new customError(id + " is not a valid padId", "apierror");
}
// check if this is a valid text
if (text != null) {
// check if text is a string
if (typeof text != "string") {
throw new customError("text is not a string", "apierror");
}
// check if text is less than 100k chars
if (text.length > 100000) {
throw new customError("text must be less than 100k chars", "apierror");
}
}
let pad = globalPads.get(id);
// return pad if it's already loaded
if (pad != null) {
return pad;
}
// try to load pad
pad = new Pad(id);
// initalize the pad
await pad.init(text);
globalPads.set(id, pad);
padList.addPad(id);
return pad;
}
exports.listAllPads = async function()
{
let padIDs = await padList.getPads();
return { padIDs };
}
// checks if a pad exists
exports.doesPadExist = async function(padId)
{
let value = await db.get("pad:" + padId);
return (value != null && value.atext);
}
// alias for backwards compatibility
exports.doesPadExists = exports.doesPadExist;
/** /**
* An array of padId transformations. These represent changes in pad name policy over * An array of padId transformations. These represent changes in pad name policy over
* time, and allow us to "play back" these changes so legacy padIds can be found. * time, and allow us to "play back" these changes so legacy padIds can be found.
*/ */
var padIdTransforms = [ const padIdTransforms = [
[/\s+/g, '_'], [/\s+/g, '_'],
[/:+/g, '_'] [/:+/g, '_']
]; ];
/** // returns a sanitized padId, respecting legacy pad id formats
* Returns a Pad Object with the callback exports.sanitizePadId = async function sanitizePadId(padId) {
* @param id A String with the id of the pad for (let i = 0, n = padIdTransforms.length; i < n; ++i) {
* @param {Function} callback let exists = await exports.doesPadExist(padId);
*/
exports.getPad = function(id, text, callback) if (exists) {
{ return padId;
//check if this is a valid padId
if(!exports.isValidPadId(id))
{
callback(new customError(id + " is not a valid padId","apierror"));
return;
}
//make text an optional parameter
if(typeof text == "function")
{
callback = text;
text = null;
}
//check if this is a valid text
if(text != null)
{
//check if text is a string
if(typeof text != "string")
{
callback(new customError("text is not a string","apierror"));
return;
} }
//check if text is less than 100k chars let [from, to] = padIdTransforms[i];
if(text.length > 100000)
{ padId = padId.replace(from, to);
callback(new customError("text must be less than 100k chars","apierror"));
return;
}
}
var pad = globalPads.get(id);
//return pad if its already loaded
if(pad != null)
{
callback(null, pad);
return;
} }
//try to load pad // we're out of possible transformations, so just return it
pad = new Pad(id); return padId;
//initalize the pad
pad.init(text, function(err)
{
if(ERR(err, callback)) return;
globalPads.set(id, pad);
padList.addPad(id);
callback(null, pad);
});
}
exports.listAllPads = function(cb)
{
padList.getPads(function(list) {
cb && cb(null, {padIDs: list});
});
}
//checks if a pad exists
exports.doesPadExists = function(padId, callback)
{
db.get("pad:"+padId, function(err, value)
{
if(ERR(err, callback)) return;
if(value != null && value.atext){
callback(null, true);
}
else
{
callback(null, false);
}
});
}
//returns a sanitized padId, respecting legacy pad id formats
exports.sanitizePadId = function(padId, callback) {
var transform_index = arguments[2] || 0;
//we're out of possible transformations, so just return it
if(transform_index >= padIdTransforms.length)
{
callback(padId);
return;
}
//check if padId exists
exports.doesPadExists(padId, function(junk, exists)
{
if(exists)
{
callback(padId);
return;
}
//get the next transformation *that's different*
var transformedPadId = padId;
while(transformedPadId == padId && transform_index < padIdTransforms.length)
{
transformedPadId = padId.replace(padIdTransforms[transform_index][0], padIdTransforms[transform_index][1]);
transform_index += 1;
}
//check the next transform
exports.sanitizePadId(transformedPadId, callback, transform_index);
});
} }
exports.isValidPadId = function(padId) exports.isValidPadId = function(padId)
@ -237,13 +203,13 @@ exports.isValidPadId = function(padId)
/** /**
* Removes the pad from database and unloads it. * Removes the pad from database and unloads it.
*/ */
exports.removePad = function(padId){ exports.removePad = function(padId) {
db.remove("pad:"+padId); db.remove("pad:" + padId);
exports.unloadPad(padId); exports.unloadPad(padId);
padList.removePad(padId); padList.removePad(padId);
} }
//removes a pad from the cache // removes a pad from the cache
exports.unloadPad = function(padId) exports.unloadPad = function(padId)
{ {
globalPads.remove(padId); globalPads.remove(padId);

85
src/node/db/ReadOnlyManager.js
View file

@ -19,80 +19,47 @@
*/ */
var ERR = require("async-stacktrace"); var db = require("./DB");
var db = require("./DB").db;
var async = require("async");
var randomString = require("../utils/randomstring"); var randomString = require("../utils/randomstring");
/** /**
* returns a read only id for a pad * returns a read only id for a pad
* @param {String} padId the id of the pad * @param {String} padId the id of the pad
*/ */
exports.getReadOnlyId = function (padId, callback) exports.getReadOnlyId = async function (padId)
{ {
var readOnlyId; // check if there is a pad2readonly entry
let readOnlyId = await db.get("pad2readonly:" + padId);
async.waterfall([
//check if there is a pad2readonly entry // there is no readOnly Entry in the database, let's create one
function(callback) if (readOnlyId == null) {
{ readOnlyId = "r." + randomString(16);
db.get("pad2readonly:" + padId, callback); db.set("pad2readonly:" + padId, readOnlyId);
}, db.set("readonly2pad:" + readOnlyId, padId);
function(dbReadOnlyId, callback) }
{
//there is no readOnly Entry in the database, let's create one return readOnlyId;
if(dbReadOnlyId == null)
{
readOnlyId = "r." + randomString(16);
db.set("pad2readonly:" + padId, readOnlyId);
db.set("readonly2pad:" + readOnlyId, padId);
}
//there is a readOnly Entry in the database, let's take this one
else
{
readOnlyId = dbReadOnlyId;
}
callback();
}
], function(err)
{
if(ERR(err, callback)) return;
//return the results
callback(null, readOnlyId);
})
} }
/** /**
* returns a the padId for a read only id * returns the padId for a read only id
* @param {String} readOnlyId read only id * @param {String} readOnlyId read only id
*/ */
exports.getPadId = function(readOnlyId, callback) exports.getPadId = function(readOnlyId)
{ {
db.get("readonly2pad:" + readOnlyId, callback); return db.get("readonly2pad:" + readOnlyId);
} }
/** /**
* returns a the padId and readonlyPadId in an object for any id * returns the padId and readonlyPadId in an object for any id
* @param {String} padIdOrReadonlyPadId read only id or real pad id * @param {String} padIdOrReadonlyPadId read only id or real pad id
*/ */
exports.getIds = function(id, callback) { exports.getIds = async function(id) {
if (id.indexOf("r.") == 0) let readonly = (id.indexOf("r.") === 0);
exports.getPadId(id, function (err, value) {
if(ERR(err, callback)) return; // Might be null, if this is an unknown read-only id
callback(null, { let readOnlyPadId = readonly ? id : await exports.getReadOnlyId(id);
readOnlyPadId: id, let padId = readonly ? await exports.getPadId(id) : id;
padId: value, // Might be null, if this is an unknown read-only id
readonly: true return { readOnlyPadId, padId, readonly };
});
});
else
exports.getReadOnlyId(id, function (err, value) {
callback(null, {
readOnlyPadId: value,
padId: id,
readonly: false
});
});
} }

496
src/node/db/SecurityManager.js
View file

@ -18,9 +18,6 @@
* limitations under the License. * limitations under the License.
*/ */
var ERR = require("async-stacktrace");
var async = require("async");
var authorManager = require("./AuthorManager"); var authorManager = require("./AuthorManager");
var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js"); var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js");
var padManager = require("./PadManager"); var padManager = require("./PadManager");
@ -34,296 +31,231 @@ var authLogger = log4js.getLogger("auth");
* @param padID the pad the user wants to access * @param padID the pad the user wants to access
* @param sessionCookie the session the user has (set via api) * @param sessionCookie the session the user has (set via api)
* @param token the token of the author (randomly generated at client side, used for public pads) * @param token the token of the author (randomly generated at client side, used for public pads)
* @param password the password the user has given to access this pad, can be null * @param password the password the user has given to access this pad, can be null
* @param callback will be called with (err, {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx}) * @return {accessStatus: grant|deny|wrongPassword|needPassword, authorID: a.xxxxxx})
*/ */
exports.checkAccess = function (padID, sessionCookie, token, password, callback) exports.checkAccess = async function(padID, sessionCookie, token, password)
{ {
var statusObject; // immutable object
let deny = Object.freeze({ accessStatus: "deny" });
if(!padID) {
callback(null, {accessStatus: "deny"}); if (!padID) {
return; return deny;
} }
// allow plugins to deny access // allow plugins to deny access
var deniedByHook = hooks.callAll("onAccessCheck", {'padID': padID, 'password': password, 'token': token, 'sessionCookie': sessionCookie}).indexOf(false) > -1; var deniedByHook = hooks.callAll("onAccessCheck", {'padID': padID, 'password': password, 'token': token, 'sessionCookie': sessionCookie}).indexOf(false) > -1;
if(deniedByHook) if (deniedByHook) {
{ return deny;
callback(null, {accessStatus: "deny"});
return;
} }
// a valid session is required (api-only mode) // start to get author for this token
if(settings.requireSession) let p_tokenAuthor = authorManager.getAuthor4Token(token);
{
// without sessionCookie, access is denied // start to check if pad exists
if(!sessionCookie) let p_padExists = padManager.doesPadExist(padID);
{
callback(null, {accessStatus: "deny"}); if (settings.requireSession) {
return; // a valid session is required (api-only mode)
if (!sessionCookie) {
// without sessionCookie, access is denied
return deny;
} }
} } else {
// a session is not required, so we'll check if it's a public pad // a session is not required, so we'll check if it's a public pad
else if (padID.indexOf("$") === -1) {
{ // it's not a group pad, means we can grant access
// it's not a group pad, means we can grant access
if(padID.indexOf("$") == -1) // assume user has access
{ let authorID = await p_tokenAuthor;
//get author for this token let statusObject = { accessStatus: "grant", authorID };
authorManager.getAuthor4Token(token, function(err, author)
{ if (settings.editOnly) {
if(ERR(err, callback)) return;
// assume user has access
statusObject = {accessStatus: "grant", authorID: author};
// user can't create pads // user can't create pads
if(settings.editOnly)
{
// check if pad exists
padManager.doesPadExists(padID, function(err, exists)
{
if(ERR(err, callback)) return;
// pad doesn't exist - user can't have access
if(!exists) statusObject.accessStatus = "deny";
// grant or deny access, with author of token
callback(null, statusObject);
});
return; let padExists = await p_padExists;
}
// user may create new pads - no need to check anything if (!padExists) {
// grant access, with author of token // pad doesn't exist - user can't have access
callback(null, statusObject);
});
//don't continue
return;
}
}
var groupID = padID.split("$")[0];
var padExists = false;
var validSession = false;
var sessionAuthor;
var tokenAuthor;
var isPublic;
var isPasswordProtected;
var passwordStatus = password == null ? "notGiven" : "wrong"; // notGiven, correct, wrong
async.series([
//get basic informations from the database
function(callback)
{
async.parallel([
//does pad exists
function(callback)
{
padManager.doesPadExists(padID, function(err, exists)
{
if(ERR(err, callback)) return;
padExists = exists;
callback();
});
},
//get information about all sessions contained in this cookie
function(callback)
{
if (!sessionCookie)
{
callback();
return;
}
var sessionIDs = sessionCookie.split(',');
async.forEach(sessionIDs, function(sessionID, callback)
{
sessionManager.getSessionInfo(sessionID, function(err, sessionInfo)
{
//skip session if it doesn't exist
if(err && err.message == "sessionID does not exist")
{
authLogger.debug("Auth failed: unknown session");
callback();
return;
}
if(ERR(err, callback)) return;
var now = Math.floor(Date.now()/1000);
//is it for this group?
if(sessionInfo.groupID != groupID)
{
authLogger.debug("Auth failed: wrong group");
callback();
return;
}
//is validUntil still ok?
if(sessionInfo.validUntil <= now)
{
authLogger.debug("Auth failed: validUntil");
callback();
return;
}
// There is a valid session
validSession = true;
sessionAuthor = sessionInfo.authorID;
callback();
});
}, callback);
},
//get author for token
function(callback)
{
//get author for this token
authorManager.getAuthor4Token(token, function(err, author)
{
if(ERR(err, callback)) return;
tokenAuthor = author;
callback();
});
}
], callback);
},
//get more informations of this pad, if avaiable
function(callback)
{
//skip this if the pad doesn't exists
if(padExists == false)
{
callback();
return;
}
padManager.getPad(padID, function(err, pad)
{
if(ERR(err, callback)) return;
//is it a public pad?
isPublic = pad.getPublicStatus();
//is it password protected?
isPasswordProtected = pad.isPasswordProtected();
//is password correct?
if(isPasswordProtected && password && pad.isCorrectPassword(password))
{
passwordStatus = "correct";
}
callback();
});
},
function(callback)
{
//- a valid session for this group is avaible AND pad exists
if(validSession && padExists)
{
//- the pad is not password protected
if(!isPasswordProtected)
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
}
//- the setting to bypass password validation is set
else if(settings.sessionNoPassword)
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
}
//- the pad is password protected and password is correct
else if(isPasswordProtected && passwordStatus == "correct")
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
}
//- the pad is password protected but wrong password given
else if(isPasswordProtected && passwordStatus == "wrong")
{
//--> deny access, ask for new password and tell them that the password is wrong
statusObject = {accessStatus: "wrongPassword"};
}
//- the pad is password protected but no password given
else if(isPasswordProtected && passwordStatus == "notGiven")
{
//--> ask for password
statusObject = {accessStatus: "needPassword"};
}
else
{
throw new Error("Ops, something wrong happend");
}
}
//- a valid session for this group avaible but pad doesn't exists
else if(validSession && !padExists)
{
//--> grant access
statusObject = {accessStatus: "grant", authorID: sessionAuthor};
//--> deny access if user isn't allowed to create the pad
if(settings.editOnly)
{
authLogger.debug("Auth failed: valid session & pad does not exist");
statusObject.accessStatus = "deny"; statusObject.accessStatus = "deny";
} }
} }
// there is no valid session avaiable AND pad exists
else if(!validSession && padExists) // user may create new pads - no need to check anything
{ // grant access, with author of token
//-- its public and not password protected return statusObject;
if(isPublic && !isPasswordProtected)
{
//--> grant access, with author of token
statusObject = {accessStatus: "grant", authorID: tokenAuthor};
}
//- its public and password protected and password is correct
else if(isPublic && isPasswordProtected && passwordStatus == "correct")
{
//--> grant access, with author of token
statusObject = {accessStatus: "grant", authorID: tokenAuthor};
}
//- its public and the pad is password protected but wrong password given
else if(isPublic && isPasswordProtected && passwordStatus == "wrong")
{
//--> deny access, ask for new password and tell them that the password is wrong
statusObject = {accessStatus: "wrongPassword"};
}
//- its public and the pad is password protected but no password given
else if(isPublic && isPasswordProtected && passwordStatus == "notGiven")
{
//--> ask for password
statusObject = {accessStatus: "needPassword"};
}
//- its not public
else if(!isPublic)
{
authLogger.debug("Auth failed: invalid session & pad is not public");
//--> deny access
statusObject = {accessStatus: "deny"};
}
else
{
throw new Error("Ops, something wrong happend");
}
}
// there is no valid session avaiable AND pad doesn't exists
else
{
authLogger.debug("Auth failed: invalid session & pad does not exist");
//--> deny access
statusObject = {accessStatus: "deny"};
}
callback();
} }
], function(err) }
{
if(ERR(err, callback)) return; let validSession = false;
callback(null, statusObject); let sessionAuthor;
}); let isPublic;
}; let isPasswordProtected;
let passwordStatus = password == null ? "notGiven" : "wrong"; // notGiven, correct, wrong
// get information about all sessions contained in this cookie
if (sessionCookie) {
let groupID = padID.split("$")[0];
let sessionIDs = sessionCookie.split(',');
// was previously iterated in parallel using async.forEach
let sessionInfos = await Promise.all(sessionIDs.map(sessionID => {
return sessionManager.getSessionInfo(sessionID);
}));
// seperated out the iteration of sessioninfos from the (parallel) fetches from the DB
for (let sessionInfo of sessionInfos) {
try {
// is it for this group?
if (sessionInfo.groupID != groupID) {
authLogger.debug("Auth failed: wrong group");
continue;
}
// is validUntil still ok?
let now = Math.floor(Date.now() / 1000);
if (sessionInfo.validUntil <= now) {
authLogger.debug("Auth failed: validUntil");
continue;
}
// fall-through - there is a valid session
validSession = true;
sessionAuthor = sessionInfo.authorID;
break;
} catch (err) {
// skip session if it doesn't exist
if (err.message == "sessionID does not exist") {
authLogger.debug("Auth failed: unknown session");
} else {
throw err;
}
}
}
}
let padExists = await p_padExists;
if (padExists) {
let pad = await padManager.getPad(padID);
// is it a public pad?
isPublic = pad.getPublicStatus();
// is it password protected?
isPasswordProtected = pad.isPasswordProtected();
// is password correct?
if (isPasswordProtected && password && pad.isCorrectPassword(password)) {
passwordStatus = "correct";
}
}
// - a valid session for this group is avaible AND pad exists
if (validSession && padExists) {
let authorID = sessionAuthor;
let grant = Object.freeze({ accessStatus: "grant", authorID });
if (!isPasswordProtected) {
// - the pad is not password protected
// --> grant access
return grant;
}
if (settings.sessionNoPassword) {
// - the setting to bypass password validation is set
// --> grant access
return grant;
}
if (isPasswordProtected && passwordStatus === "correct") {
// - the pad is password protected and password is correct
// --> grant access
return grant;
}
if (isPasswordProtected && passwordStatus === "wrong") {
// - the pad is password protected but wrong password given
// --> deny access, ask for new password and tell them that the password is wrong
return { accessStatus: "wrongPassword" };
}
if (isPasswordProtected && passwordStatus === "notGiven") {
// - the pad is password protected but no password given
// --> ask for password
return { accessStatus: "needPassword" };
}
throw new Error("Oops, something wrong happend");
}
if (validSession && !padExists) {
// - a valid session for this group avaible but pad doesn't exist
// --> grant access by default
let accessStatus = "grant";
let authorID = sessionAuthor;
// --> deny access if user isn't allowed to create the pad
if (settings.editOnly) {
authLogger.debug("Auth failed: valid session & pad does not exist");
accessStatus = "deny";
}
return { accessStatus, authorID };
}
if (!validSession && padExists) {
// there is no valid session avaiable AND pad exists
let authorID = await p_tokenAuthor;
let grant = Object.freeze({ accessStatus: "grant", authorID });
if (isPublic && !isPasswordProtected) {
// -- it's public and not password protected
// --> grant access, with author of token
return grant;
}
if (isPublic && isPasswordProtected && passwordStatus === "correct") {
// - it's public and password protected and password is correct
// --> grant access, with author of token
return grant;
}
if (isPublic && isPasswordProtected && passwordStatus === "wrong") {
// - it's public and the pad is password protected but wrong password given
// --> deny access, ask for new password and tell them that the password is wrong
return { accessStatus: "wrongPassword" };
}
if (isPublic && isPasswordProtected && passwordStatus === "notGiven") {
// - it's public and the pad is password protected but no password given
// --> ask for password
return { accessStatus: "needPassword" };
}
if (!isPublic) {
// - it's not public
authLogger.debug("Auth failed: invalid session & pad is not public");
// --> deny access
return { accessStatus: "deny" };
}
throw new Error("Oops, something wrong happend");
}
// there is no valid session avaiable AND pad doesn't exist
authLogger.debug("Auth failed: invalid session & pad does not exist");
return { accessStatus: "deny" };
}

495
src/node/db/SessionManager.js
View file

@ -17,361 +17,208 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
var ERR = require("async-stacktrace");
var customError = require("../utils/customError"); var customError = require("../utils/customError");
var randomString = require("../utils/randomstring"); var randomString = require("../utils/randomstring");
var db = require("./DB").db; var db = require("./DB");
var async = require("async"); var groupManager = require("./GroupManager");
var groupMangager = require("./GroupManager"); var authorManager = require("./AuthorManager");
var authorMangager = require("./AuthorManager");
exports.doesSessionExist = async function(sessionID)
exports.doesSessionExist = function(sessionID, callback)
{ {
//check if the database entry of this session exists //check if the database entry of this session exists
db.get("session:" + sessionID, function (err, session) let session = await db.get("session:" + sessionID);
{ return (session !== null);
if(ERR(err, callback)) return;
callback(null, session != null);
});
} }
/** /**
* Creates a new session between an author and a group * Creates a new session between an author and a group
*/ */
exports.createSession = function(groupID, authorID, validUntil, callback) exports.createSession = async function(groupID, authorID, validUntil)
{ {
var sessionID; // check if the group exists
let groupExists = await groupManager.doesGroupExist(groupID);
if (!groupExists) {
throw new customError("groupID does not exist", "apierror");
}
async.series([ // check if the author exists
//check if group exists let authorExists = await authorManager.doesAuthorExist(authorID);
function(callback) if (!authorExists) {
{ throw new customError("authorID does not exist", "apierror");
groupMangager.doesGroupExist(groupID, function(err, exists) }
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("groupID does not exist","apierror"));
}
//everything is fine, continue
else
{
callback();
}
});
},
//check if author exists
function(callback)
{
authorMangager.doesAuthorExists(authorID, function(err, exists)
{
if(ERR(err, callback)) return;
//author does not exist
if(exists == false)
{
callback(new customError("authorID does not exist","apierror"));
}
//everything is fine, continue
else
{
callback();
}
});
},
//check validUntil and create the session db entry
function(callback)
{
//check if rev is a number
if(typeof validUntil != "number")
{
//try to parse the number
if(isNaN(parseInt(validUntil)))
{
callback(new customError("validUntil is not a number","apierror"));
return;
}
validUntil = parseInt(validUntil); // try to parse validUntil if it's not a number
} if (typeof validUntil !== "number") {
validUntil = parseInt(validUntil);
//ensure this is not a negativ number }
if(validUntil < 0)
{ // check it's a valid number
callback(new customError("validUntil is a negativ number","apierror")); if (isNaN(validUntil)) {
return; throw new customError("validUntil is not a number", "apierror");
} }
//ensure this is not a float value // ensure this is not a negative number
if(!is_int(validUntil)) if (validUntil < 0) {
{ throw new customError("validUntil is a negative number", "apierror");
callback(new customError("validUntil is a float value","apierror")); }
return;
} // ensure this is not a float value
if (!is_int(validUntil)) {
//check if validUntil is in the future throw new customError("validUntil is a float value", "apierror");
if(Math.floor(Date.now()/1000) > validUntil) }
{
callback(new customError("validUntil is in the past","apierror")); // check if validUntil is in the future
return; if (validUntil < Math.floor(Date.now() / 1000)) {
} throw new customError("validUntil is in the past", "apierror");
}
//generate sessionID
sessionID = "s." + randomString(16); // generate sessionID
let sessionID = "s." + randomString(16);
//set the session into the database
db.set("session:" + sessionID, {"groupID": groupID, "authorID": authorID, "validUntil": validUntil}); // set the session into the database
await db.set("session:" + sessionID, {"groupID": groupID, "authorID": authorID, "validUntil": validUntil});
callback();
}, // get the entry
//set the group2sessions entry let group2sessions = await db.get("group2sessions:" + groupID);
function(callback)
{ /*
//get the entry * In some cases, the db layer could return "undefined" as well as "null".
db.get("group2sessions:" + groupID, function(err, group2sessions) * Thus, it is not possible to perform strict null checks on group2sessions.
{ * In a previous version of this code, a strict check broke session
if(ERR(err, callback)) return; * management.
*
//the entry doesn't exist so far, let's create it * See: https://github.com/ether/etherpad-lite/issues/3567#issuecomment-468613960
if(group2sessions == null || group2sessions.sessionIDs == null) */
{ if (!group2sessions || !group2sessions.sessionIDs) {
group2sessions = {sessionIDs : {}}; // the entry doesn't exist so far, let's create it
} group2sessions = {sessionIDs : {}};
}
//add the entry for this session
group2sessions.sessionIDs[sessionID] = 1; // add the entry for this session
group2sessions.sessionIDs[sessionID] = 1;
//save the new element back
db.set("group2sessions:" + groupID, group2sessions); // save the new element back
await db.set("group2sessions:" + groupID, group2sessions);
callback();
}); // get the author2sessions entry
}, let author2sessions = await db.get("author2sessions:" + authorID);
//set the author2sessions entry
function(callback) if (author2sessions == null || author2sessions.sessionIDs == null) {
{ // the entry doesn't exist so far, let's create it
//get the entry author2sessions = {sessionIDs : {}};
db.get("author2sessions:" + authorID, function(err, author2sessions) }
{
if(ERR(err, callback)) return; // add the entry for this session
author2sessions.sessionIDs[sessionID] = 1;
//the entry doesn't exist so far, let's create it
if(author2sessions == null || author2sessions.sessionIDs == null) //save the new element back
{ await db.set("author2sessions:" + authorID, author2sessions);
author2sessions = {sessionIDs : {}};
} return { sessionID };
//add the entry for this session
author2sessions.sessionIDs[sessionID] = 1;
//save the new element back
db.set("author2sessions:" + authorID, author2sessions);
callback();
});
}
], function(err)
{
if(ERR(err, callback)) return;
//return error and sessionID
callback(null, {sessionID: sessionID});
})
} }
exports.getSessionInfo = function(sessionID, callback) exports.getSessionInfo = async function(sessionID)
{ {
//check if the database entry of this session exists // check if the database entry of this session exists
db.get("session:" + sessionID, function (err, session) let session = await db.get("session:" + sessionID);
{
if(ERR(err, callback)) return; if (session == null) {
// session does not exist
//session does not exists throw new customError("sessionID does not exist", "apierror");
if(session == null) }
{
callback(new customError("sessionID does not exist","apierror")) // everything is fine, return the sessioninfos
} return session;
//everything is fine, return the sessioninfos
else
{
callback(null, session);
}
});
} }
/** /**
* Deletes a session * Deletes a session
*/ */
exports.deleteSession = function(sessionID, callback) exports.deleteSession = async function(sessionID)
{ {
var authorID, groupID; // ensure that the session exists
var group2sessions, author2sessions; let session = await db.get("session:" + sessionID);
if (session == null) {
throw new customError("sessionID does not exist", "apierror");
}
async.series([ // everything is fine, use the sessioninfos
function(callback) let groupID = session.groupID;
{ let authorID = session.authorID;
//get the session entry
db.get("session:" + sessionID, function (err, session)
{
if(ERR(err, callback)) return;
//session does not exists
if(session == null)
{
callback(new customError("sessionID does not exist","apierror"))
}
//everything is fine, return the sessioninfos
else
{
authorID = session.authorID;
groupID = session.groupID;
callback();
}
});
},
//get the group2sessions entry
function(callback)
{
db.get("group2sessions:" + groupID, function (err, _group2sessions)
{
if(ERR(err, callback)) return;
group2sessions = _group2sessions;
callback();
});
},
//get the author2sessions entry
function(callback)
{
db.get("author2sessions:" + authorID, function (err, _author2sessions)
{
if(ERR(err, callback)) return;
author2sessions = _author2sessions;
callback();
});
},
//remove the values from the database
function(callback)
{
//remove the session
db.remove("session:" + sessionID);
//remove session from group2sessions
if(group2sessions != null) { // Maybe the group was already deleted
delete group2sessions.sessionIDs[sessionID];
db.set("group2sessions:" + groupID, group2sessions);
}
//remove session from author2sessions // get the group2sessions and author2sessions entries
if(author2sessions != null) { // Maybe the author was already deleted let group2sessions = await db.get("group2sessions:" + groupID);
delete author2sessions.sessionIDs[sessionID]; let author2sessions = await db.get("author2sessions:" + authorID);
db.set("author2sessions:" + authorID, author2sessions);
} // remove the session
await db.remove("session:" + sessionID);
callback();
} // remove session from group2sessions
], function(err) if (group2sessions != null) { // Maybe the group was already deleted
{ delete group2sessions.sessionIDs[sessionID];
if(ERR(err, callback)) return; await db.set("group2sessions:" + groupID, group2sessions);
callback(); }
})
// remove session from author2sessions
if (author2sessions != null) { // Maybe the author was already deleted
delete author2sessions.sessionIDs[sessionID];
await db.set("author2sessions:" + authorID, author2sessions);
}
} }
exports.listSessionsOfGroup = function(groupID, callback) exports.listSessionsOfGroup = async function(groupID)
{ {
groupMangager.doesGroupExist(groupID, function(err, exists) // check that the group exists
{ let exists = await groupManager.doesGroupExist(groupID);
if(ERR(err, callback)) return; if (!exists) {
throw new customError("groupID does not exist", "apierror");
//group does not exist }
if(exists == false)
{ let sessions = await listSessionsWithDBKey("group2sessions:" + groupID);
callback(new customError("groupID does not exist","apierror")); return sessions;
}
//everything is fine, continue
else
{
listSessionsWithDBKey("group2sessions:" + groupID, callback);
}
});
} }
exports.listSessionsOfAuthor = function(authorID, callback) exports.listSessionsOfAuthor = async function(authorID)
{
authorMangager.doesAuthorExists(authorID, function(err, exists)
{
if(ERR(err, callback)) return;
//group does not exist
if(exists == false)
{
callback(new customError("authorID does not exist","apierror"));
}
//everything is fine, continue
else
{
listSessionsWithDBKey("author2sessions:" + authorID, callback);
}
});
}
//this function is basicly the code listSessionsOfAuthor and listSessionsOfGroup has in common
function listSessionsWithDBKey (dbkey, callback)
{ {
var sessions; // check that the author exists
let exists = await authorManager.doesAuthorExist(authorID)
if (!exists) {
throw new customError("authorID does not exist", "apierror");
}
async.series([ let sessions = await listSessionsWithDBKey("author2sessions:" + authorID);
function(callback) return sessions;
{
//get the group2sessions entry
db.get(dbkey, function(err, sessionObject)
{
if(ERR(err, callback)) return;
sessions = sessionObject ? sessionObject.sessionIDs : null;
callback();
});
},
function(callback)
{
//collect all sessionIDs in an arrary
var sessionIDs = [];
for (var i in sessions)
{
sessionIDs.push(i);
}
//foreach trough the sessions and get the sessioninfos
async.forEach(sessionIDs, function(sessionID, callback)
{
exports.getSessionInfo(sessionID, function(err, sessionInfo)
{
if (err == "apierror: sessionID does not exist")
{
console.warn(`Found bad session ${sessionID} in ${dbkey}`);
}
else if(ERR(err, callback))
{
return;
}
sessions[sessionID] = sessionInfo;
callback();
});
}, callback);
}
], function(err)
{
if(ERR(err, callback)) return;
callback(null, sessions);
});
} }
//checks if a number is an int // this function is basically the code listSessionsOfAuthor and listSessionsOfGroup has in common
// required to return null rather than an empty object if there are none
async function listSessionsWithDBKey(dbkey)
{
// get the group2sessions entry
let sessionObject = await db.get(dbkey);
let sessions = sessionObject ? sessionObject.sessionIDs : null;
// iterate through the sessions and get the sessioninfos
for (let sessionID in sessions) {
try {
let sessionInfo = await exports.getSessionInfo(sessionID);
sessions[sessionID] = sessionInfo;
} catch (err) {
if (err == "apierror: sessionID does not exist") {
console.warn(`Found bad session ${sessionID} in ${dbkey}`);
sessions[sessionID] = null;
} else {
throw err;
}
}
}
return sessions;
}
// checks if a number is an int
function is_int(value) function is_int(value)
{ {
return (parseFloat(value) == parseInt(value)) && !isNaN(value) return (parseFloat(value) == parseInt(value)) && !isNaN(value);
} }

98
src/node/db/SessionStore.js
View file

@ -1,7 +1,10 @@
/* /*
* Stores session data in the database * Stores session data in the database
* Source; https://github.com/edy-b/SciFlowWriter/blob/develop/available_plugins/ep_sciflowwriter/db/DirtyStore.js * Source; https://github.com/edy-b/SciFlowWriter/blob/develop/available_plugins/ep_sciflowwriter/db/DirtyStore.js
* This is not used for authors that are created via the API at current * This is not used for authors that are created via the API at current
*
* RPB: this module was not migrated to Promises, because it is only used via
* express-session, which can't actually use promises anyway.
*/ */
var Store = require('ep_etherpad-lite/node_modules/express-session').Store, var Store = require('ep_etherpad-lite/node_modules/express-session').Store,
@ -13,11 +16,12 @@ var SessionStore = module.exports = function SessionStore() {};
SessionStore.prototype.__proto__ = Store.prototype; SessionStore.prototype.__proto__ = Store.prototype;
SessionStore.prototype.get = function(sid, fn){ SessionStore.prototype.get = function(sid, fn) {
messageLogger.debug('GET ' + sid); messageLogger.debug('GET ' + sid);
var self = this; var self = this;
db.get("sessionstorage:" + sid, function (err, sess)
{ db.get("sessionstorage:" + sid, function(err, sess) {
if (sess) { if (sess) {
sess.cookie.expires = 'string' == typeof sess.cookie.expires ? new Date(sess.cookie.expires) : sess.cookie.expires; sess.cookie.expires = 'string' == typeof sess.cookie.expires ? new Date(sess.cookie.expires) : sess.cookie.expires;
if (!sess.cookie.expires || new Date() < sess.cookie.expires) { if (!sess.cookie.expires || new Date() < sess.cookie.expires) {
@ -31,50 +35,64 @@ SessionStore.prototype.get = function(sid, fn){
}); });
}; };
SessionStore.prototype.set = function(sid, sess, fn){ SessionStore.prototype.set = function(sid, sess, fn) {
messageLogger.debug('SET ' + sid); messageLogger.debug('SET ' + sid);
db.set("sessionstorage:" + sid, sess); db.set("sessionstorage:" + sid, sess);
process.nextTick(function(){ if (fn) {
if(fn) fn(); process.nextTick(fn);
}); }
}; };
SessionStore.prototype.destroy = function(sid, fn){ SessionStore.prototype.destroy = function(sid, fn) {
messageLogger.debug('DESTROY ' + sid); messageLogger.debug('DESTROY ' + sid);
db.remove("sessionstorage:" + sid); db.remove("sessionstorage:" + sid);
process.nextTick(function(){ if (fn) {
if(fn) fn(); process.nextTick(fn);
}); }
}; };
SessionStore.prototype.all = function(fn){ /*
messageLogger.debug('ALL'); * RPB: the following methods are optional requirements for a compatible session
var sessions = []; * store for express-session, but in any case appear to depend on a
db.forEach(function(key, value){ * non-existent feature of ueberdb2
if (key.substr(0,15) === "sessionstorage:") { */
sessions.push(value); if (db.forEach) {
} SessionStore.prototype.all = function(fn) {
}); messageLogger.debug('ALL');
fn(null, sessions);
};
SessionStore.prototype.clear = function(fn){ var sessions = [];
messageLogger.debug('CLEAR');
db.forEach(function(key, value){
if (key.substr(0,15) === "sessionstorage:") {
db.db.remove("session:" + key);
}
});
if(fn) fn();
};
SessionStore.prototype.length = function(fn){ db.forEach(function(key, value) {
messageLogger.debug('LENGTH'); if (key.substr(0,15) === "sessionstorage:") {
var i = 0; sessions.push(value);
db.forEach(function(key, value){ }
if (key.substr(0,15) === "sessionstorage:") { });
i++; fn(null, sessions);
} };
});
fn(null, i); SessionStore.prototype.clear = function(fn) {
messageLogger.debug('CLEAR');
db.forEach(function(key, value) {
if (key.substr(0,15) === "sessionstorage:") {
db.remove("session:" + key);
}
});
if (fn) fn();
};
SessionStore.prototype.length = function(fn) {
messageLogger.debug('LENGTH');
var i = 0;
db.forEach(function(key, value) {
if (key.substr(0,15) === "sessionstorage:") {
i++;
}
});
fn(null, i);
}
}; };

134
src/node/handler/APIHandler.js
View file

@ -19,7 +19,6 @@
*/ */
var absolutePaths = require('../utils/AbsolutePaths'); var absolutePaths = require('../utils/AbsolutePaths');
var ERR = require("async-stacktrace");
var fs = require("fs"); var fs = require("fs");
var api = require("../db/API"); var api = require("../db/API");
var log4js = require('log4js'); var log4js = require('log4js');
@ -32,19 +31,17 @@ var apiHandlerLogger = log4js.getLogger('APIHandler');
//ensure we have an apikey //ensure we have an apikey
var apikey = null; var apikey = null;
var apikeyFilename = absolutePaths.makeAbsolute(argv.apikey || "./APIKEY.txt"); var apikeyFilename = absolutePaths.makeAbsolute(argv.apikey || "./APIKEY.txt");
try
{ try {
apikey = fs.readFileSync(apikeyFilename,"utf8"); apikey = fs.readFileSync(apikeyFilename,"utf8");
apiHandlerLogger.info(`Api key file read from: "${apikeyFilename}"`); apiHandlerLogger.info(`Api key file read from: "${apikeyFilename}"`);
} } catch(e) {
catch(e)
{
apiHandlerLogger.info(`Api key file "${apikeyFilename}" not found. Creating with random contents.`); apiHandlerLogger.info(`Api key file "${apikeyFilename}" not found. Creating with random contents.`);
apikey = randomString(32); apikey = randomString(32);
fs.writeFileSync(apikeyFilename,apikey,"utf8"); fs.writeFileSync(apikeyFilename,apikey,"utf8");
} }
//a list of all functions // a list of all functions
var version = {}; var version = {};
version["1"] = Object.assign({}, version["1"] = Object.assign({},
@ -152,110 +149,73 @@ exports.version = version;
* @req express request object * @req express request object
* @res express response object * @res express response object
*/ */
exports.handle = function(apiVersion, functionName, fields, req, res) exports.handle = async function(apiVersion, functionName, fields, req, res)
{ {
//check if this is a valid apiversion // say goodbye if this is an unknown API version
var isKnownApiVersion = false; if (!(apiVersion in version)) {
for(var knownApiVersion in version)
{
if(knownApiVersion == apiVersion)
{
isKnownApiVersion = true;
break;
}
}
//say goodbye if this is an unknown API version
if(!isKnownApiVersion)
{
res.statusCode = 404; res.statusCode = 404;
res.send({code: 3, message: "no such api version", data: null}); res.send({code: 3, message: "no such api version", data: null});
return; return;
} }
//check if this is a valid function name // say goodbye if this is an unknown function
var isKnownFunctionname = false; if (!(functionName in version[apiVersion])) {
for(var knownFunctionname in version[apiVersion]) // no status code?!
{
if(knownFunctionname == functionName)
{
isKnownFunctionname = true;
break;
}
}
//say goodbye if this is a unknown function
if(!isKnownFunctionname)
{
res.send({code: 3, message: "no such function", data: null}); res.send({code: 3, message: "no such function", data: null});
return; return;
} }
//check the api key! // check the api key!
fields["apikey"] = fields["apikey"] || fields["api_key"]; fields["apikey"] = fields["apikey"] || fields["api_key"];
if(fields["apikey"] != apikey.trim()) if (fields["apikey"] !== apikey.trim()) {
{
res.statusCode = 401; res.statusCode = 401;
res.send({code: 4, message: "no or wrong API Key", data: null}); res.send({code: 4, message: "no or wrong API Key", data: null});
return; return;
} }
//sanitize any pad id's before continuing // sanitize any padIDs before continuing
if(fields["padID"]) if (fields["padID"]) {
{ fields["padID"] = await padManager.sanitizePadId(fields["padID"]);
padManager.sanitizePadId(fields["padID"], function(padId)
{
fields["padID"] = padId;
callAPI(apiVersion, functionName, fields, req, res);
});
} }
else if(fields["padName"]) // there was an 'else' here before - removed it to ensure
{ // that this sanitize step can't be circumvented by forcing
padManager.sanitizePadId(fields["padName"], function(padId) // the first branch to be taken
{ if (fields["padName"]) {
fields["padName"] = padId; fields["padName"] = await padManager.sanitizePadId(fields["padName"]);
callAPI(apiVersion, functionName, fields, req, res);
});
}
else
{
callAPI(apiVersion, functionName, fields, req, res);
} }
// no need to await - callAPI returns a promise
return callAPI(apiVersion, functionName, fields, req, res);
} }
//calls the api function // calls the api function
function callAPI(apiVersion, functionName, fields, req, res) async function callAPI(apiVersion, functionName, fields, req, res)
{ {
//put the function parameters in an array // put the function parameters in an array
var functionParams = version[apiVersion][functionName].map(function (field) { var functionParams = version[apiVersion][functionName].map(function (field) {
return fields[field] return fields[field]
})
//add a callback function to handle the response
functionParams.push(function(err, data)
{
// no error happend, everything is fine
if(err == null)
{
if(!data)
data = null;
res.send({code: 0, message: "ok", data: data});
}
// parameters were wrong and the api stopped execution, pass the error
else if(err.name == "apierror")
{
res.send({code: 1, message: err.message, data: null});
}
//an unknown error happend
else
{
res.send({code: 2, message: "internal error", data: null});
ERR(err);
}
}); });
//call the api function try {
api[functionName].apply(this, functionParams); // call the api function
let data = await api[functionName].apply(this, functionParams);
if (!data) {
data = null;
}
res.send({code: 0, message: "ok", data: data});
} catch (err) {
if (err.name == "apierror") {
// parameters were wrong and the api stopped execution, pass the error
res.send({code: 1, message: err.message, data: null});
} else {
// an unknown error happened
res.send({code: 2, message: "internal error", data: null});
throw err;
}
}
} }

219
src/node/handler/ExportHandler.js
View file

@ -19,163 +19,122 @@
* limitations under the License. * limitations under the License.
*/ */
var ERR = require("async-stacktrace");
var exporthtml = require("../utils/ExportHtml"); var exporthtml = require("../utils/ExportHtml");
var exporttxt = require("../utils/ExportTxt"); var exporttxt = require("../utils/ExportTxt");
var exportEtherpad = require("../utils/ExportEtherpad"); var exportEtherpad = require("../utils/ExportEtherpad");
var async = require("async");
var fs = require("fs"); var fs = require("fs");
var settings = require('../utils/Settings'); var settings = require('../utils/Settings');
var os = require('os'); var os = require('os');
var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks"); var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks");
var TidyHtml = require('../utils/TidyHtml'); var TidyHtml = require('../utils/TidyHtml');
const util = require("util");
var convertor = null; const fsp_writeFile = util.promisify(fs.writeFile);
const fsp_unlink = util.promisify(fs.unlink);
//load abiword only if its enabled let convertor = null;
if(settings.abiword != null)
// load abiword only if it is enabled
if (settings.abiword != null) {
convertor = require("../utils/Abiword"); convertor = require("../utils/Abiword");
}
// Use LibreOffice if an executable has been defined in the settings // Use LibreOffice if an executable has been defined in the settings
if(settings.soffice != null) if (settings.soffice != null) {
convertor = require("../utils/LibreOffice"); convertor = require("../utils/LibreOffice");
}
const tempDirectory = os.tmpdir(); const tempDirectory = os.tmpdir();
/** /**
* do a requested export * do a requested export
*/ */
exports.doExport = function(req, res, padId, type) async function doExport(req, res, padId, type)
{ {
var fileName = padId; var fileName = padId;
// allow fileName to be overwritten by a hook, the type type is kept static for security reasons // allow fileName to be overwritten by a hook, the type type is kept static for security reasons
hooks.aCallFirst("exportFileName", padId, let hookFileName = await hooks.aCallFirst("exportFileName", padId);
function(err, hookFileName){
// if fileName is set then set it to the padId, note that fileName is returned as an array.
if(hookFileName.length) fileName = hookFileName;
//tell the browser that this is a downloadable file // if fileName is set then set it to the padId, note that fileName is returned as an array.
res.attachment(fileName + "." + type); if (hookFileName.length) {
fileName = hookFileName;
}
//if this is a plain text export, we can do this directly // tell the browser that this is a downloadable file
// We have to over engineer this because tabs are stored as attributes and not plain text res.attachment(fileName + "." + type);
if(type == "etherpad"){
exportEtherpad.getPadRaw(padId, function(err, pad){
if(!err){
res.send(pad);
// return;
}
});
}
else if(type == "txt")
{
exporttxt.getPadTXTDocument(padId, req.params.rev, function(err, txt)
{
if(!err) {
res.send(txt);
}
});
}
else
{
var html;
var randNum;
var srcFile, destFile;
async.series([ // if this is a plain text export, we can do this directly
//render the html document // We have to over engineer this because tabs are stored as attributes and not plain text
function(callback) if (type === "etherpad") {
{ let pad = await exportEtherpad.getPadRaw(padId);
exporthtml.getPadHTMLDocument(padId, req.params.rev, function(err, _html) res.send(pad);
{ } else if (type === "txt") {
if(ERR(err, callback)) return; let txt = await exporttxt.getPadTXTDocument(padId, req.params.rev);
html = _html; res.send(txt);
callback(); } else {
}); // render the html document
}, let html = await exporthtml.getPadHTMLDocument(padId, req.params.rev);
//decide what to do with the html export
function(callback)
{
//if this is a html export, we can send this from here directly
if(type == "html")
{
// do any final changes the plugin might want to make
hooks.aCallFirst("exportHTMLSend", html, function(err, newHTML){
if(newHTML.length) html = newHTML;
res.send(html);
callback("stop");
});
}
else //write the html export to a file
{
randNum = Math.floor(Math.random()*0xFFFFFFFF);
srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html";
fs.writeFile(srcFile, html, callback);
}
},
// Tidy up the exported HTML // decide what to do with the html export
function(callback)
{
//ensure html can be collected by the garbage collector
html = null;
TidyHtml.tidy(srcFile, callback); // if this is a html export, we can send this from here directly
}, if (type === "html") {
// do any final changes the plugin might want to make
//send the convert job to the convertor (abiword, libreoffice, ..) let newHTML = await hooks.aCallFirst("exportHTMLSend", html);
function(callback) if (newHTML.length) html = newHTML;
{ res.send(html);
destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type; throw "stop";
// Allow plugins to overwrite the convert in export process
hooks.aCallAll("exportConvert", {srcFile: srcFile, destFile: destFile, req: req, res: res}, function(err, result){
if(!err && result.length > 0){
// console.log("export handled by plugin", destFile);
handledByPlugin = true;
callback();
}else{
convertor.convertFile(srcFile, destFile, type, callback);
}
});
},
//send the file
function(callback)
{
res.sendFile(destFile, null, callback);
},
//clean up temporary files
function(callback)
{
async.parallel([
function(callback)
{
fs.unlink(srcFile, callback);
},
function(callback)
{
//100ms delay to accomidate for slow windows fs
if(os.type().indexOf("Windows") > -1)
{
setTimeout(function()
{
fs.unlink(destFile, callback);
}, 100);
}
else
{
fs.unlink(destFile, callback);
}
}
], callback);
}
], function(err)
{
if(err && err != "stop") ERR(err);
})
}
} }
);
}; // else write the html export to a file
let randNum = Math.floor(Math.random()*0xFFFFFFFF);
let srcFile = tempDirectory + "/etherpad_export_" + randNum + ".html";
await fsp_writeFile(srcFile, html);
// Tidy up the exported HTML
// ensure html can be collected by the garbage collector
html = null;
await TidyHtml.tidy(srcFile);
// send the convert job to the convertor (abiword, libreoffice, ..)
let destFile = tempDirectory + "/etherpad_export_" + randNum + "." + type;
// Allow plugins to overwrite the convert in export process
let result = await hooks.aCallAll("exportConvert", { srcFile, destFile, req, res });
if (result.length > 0) {
// console.log("export handled by plugin", destFile);
handledByPlugin = true;
} else {
// @TODO no Promise interface for convertors (yet)
await new Promise((resolve, reject) => {
convertor.convertFile(srcFile, destFile, type, function(err) {
err ? reject("convertFailed") : resolve();
});
});
}
// send the file
let sendFile = util.promisify(res.sendFile);
await res.sendFile(destFile, null);
// clean up temporary files
await fsp_unlink(srcFile);
// 100ms delay to accommodate for slow windows fs
if (os.type().indexOf("Windows") > -1) {
await new Promise(resolve => setTimeout(resolve, 100));
}
await fsp_unlink(destFile);
}
}
exports.doExport = function(req, res, padId, type)
{
doExport(req, res, padId, type).catch(err => {
if (err !== "stop") {
throw err;
}
});
}

458
src/node/handler/ImportHandler.js
View file

@ -20,10 +20,8 @@
* limitations under the License. * limitations under the License.
*/ */
var ERR = require("async-stacktrace") var padManager = require("../db/PadManager")
, padManager = require("../db/PadManager")
, padMessageHandler = require("./PadMessageHandler") , padMessageHandler = require("./PadMessageHandler")
, async = require("async")
, fs = require("fs") , fs = require("fs")
, path = require("path") , path = require("path")
, settings = require('../utils/Settings') , settings = require('../utils/Settings')
@ -32,303 +30,241 @@ var ERR = require("async-stacktrace")
, importHtml = require("../utils/ImportHtml") , importHtml = require("../utils/ImportHtml")
, importEtherpad = require("../utils/ImportEtherpad") , importEtherpad = require("../utils/ImportEtherpad")
, log4js = require("log4js") , log4js = require("log4js")
, hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js"); , hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js")
, util = require("util");
var convertor = null; let fsp_exists = util.promisify(fs.exists);
var exportExtension = "htm"; let fsp_rename = util.promisify(fs.rename);
let fsp_readFile = util.promisify(fs.readFile);
let fsp_unlink = util.promisify(fs.unlink)
//load abiword only if its enabled and if soffice is disabled let convertor = null;
if(settings.abiword != null && settings.soffice === null) let exportExtension = "htm";
// load abiword only if it is enabled and if soffice is disabled
if (settings.abiword != null && settings.soffice === null) {
convertor = require("../utils/Abiword"); convertor = require("../utils/Abiword");
}
//load soffice only if its enabled // load soffice only if it is enabled
if(settings.soffice != null) { if (settings.soffice != null) {
convertor = require("../utils/LibreOffice"); convertor = require("../utils/LibreOffice");
exportExtension = "html"; exportExtension = "html";
} }
const tmpDirectory = os.tmpdir(); const tmpDirectory = os.tmpdir();
/** /**
* do a requested import * do a requested import
*/ */
exports.doImport = function(req, res, padId) async function doImport(req, res, padId)
{ {
var apiLogger = log4js.getLogger("ImportHandler"); var apiLogger = log4js.getLogger("ImportHandler");
//pipe to a file // pipe to a file
//convert file to html via abiword or soffice // convert file to html via abiword or soffice
//set html in the pad // set html in the pad
var srcFile, destFile
, pad
, text
, importHandledByPlugin
, directDatabaseAccess
, useConvertor;
var randNum = Math.floor(Math.random()*0xFFFFFFFF); var randNum = Math.floor(Math.random()*0xFFFFFFFF);
// setting flag for whether to use convertor or not // setting flag for whether to use convertor or not
useConvertor = (convertor != null); let useConvertor = (convertor != null);
async.series([ let form = new formidable.IncomingForm();
//save the uploaded file to /tmp form.keepExtensions = true;
function(callback) { form.uploadDir = tmpDirectory;
var form = new formidable.IncomingForm();
form.keepExtensions = true;
form.uploadDir = tmpDirectory;
form.parse(req, function(err, fields, files) {
//the upload failed, stop at this point
if(err || files.file === undefined) {
if(err) console.warn("Uploading Error: " + err.stack);
callback("uploadFailed");
return; // locally wrapped Promise, since form.parse requires a callback
let srcFile = await new Promise((resolve, reject) => {
form.parse(req, function(err, fields, files) {
if (err || files.file === undefined) {
// the upload failed, stop at this point
if (err) {
console.warn("Uploading Error: " + err.stack);
} }
reject("uploadFailed");
//everything ok, continue
//save the path of the uploaded file
srcFile = files.file.path;
callback();
});
},
//ensure this is a file ending we know, else we change the file ending to .txt
//this allows us to accept source code files like .c or .java
function(callback) {
var fileEnding = path.extname(srcFile).toLowerCase()
, knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"]
, fileEndingKnown = (knownFileEndings.indexOf(fileEnding) > -1);
//if the file ending is known, continue as normal
if(fileEndingKnown) {
callback();
return;
} }
resolve(files.file.path);
});
});
//we need to rename this file with a .txt ending // ensure this is a file ending we know, else we change the file ending to .txt
if(settings.allowUnknownFileEnds === true){ // this allows us to accept source code files like .c or .java
var oldSrcFile = srcFile; let fileEnding = path.extname(srcFile).toLowerCase()
srcFile = path.join(path.dirname(srcFile),path.basename(srcFile, fileEnding)+".txt"); , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"]
fs.rename(oldSrcFile, srcFile, callback); , fileEndingUnknown = (knownFileEndings.indexOf(fileEnding) < 0);
}else{
console.warn("Not allowing unknown file type to be imported", fileEnding);
callback("uploadFailed");
}
},
function(callback){
destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension);
// Logic for allowing external Import Plugins if (fileEndingUnknown) {
hooks.aCallAll("import", {srcFile: srcFile, destFile: destFile}, function(err, result){ // the file ending is not known
if(ERR(err, callback)) return callback();
if(result.length > 0){ // This feels hacky and wrong..
importHandledByPlugin = true;
}
callback();
});
},
function(callback) {
var fileEnding = path.extname(srcFile).toLowerCase()
var fileIsNotEtherpad = (fileEnding !== ".etherpad");
if (fileIsNotEtherpad) { if (settings.allowUnknownFileEnds === true) {
callback(); // we need to rename this file with a .txt ending
let oldSrcFile = srcFile;
return; srcFile = path.join(path.dirname(srcFile), path.basename(srcFile, fileEnding) + ".txt");
} await fs.rename(oldSrcFile, srcFile);
} else {
console.warn("Not allowing unknown file type to be imported", fileEnding);
throw "uploadFailed";
}
}
// we do this here so we can see if the pad has quit ea few edits let destFile = path.join(tmpDirectory, "etherpad_import_" + randNum + "." + exportExtension);
padManager.getPad(padId, function(err, _pad){
var headCount = _pad.head;
if(headCount >= 10){
apiLogger.warn("Direct database Import attempt of a pad that already has content, we wont be doing this")
return callback("padHasData");
}
fs.readFile(srcFile, "utf8", function(err, _text){ // Logic for allowing external Import Plugins
directDatabaseAccess = true; let result = await hooks.aCallAll("import", { srcFile, destFile });
importEtherpad.setPadRaw(padId, _text, function(err){ let importHandledByPlugin = (result.length > 0); // This feels hacky and wrong..
callback();
});
});
});
},
//convert file to html
function(callback) {
if (importHandledByPlugin || directDatabaseAccess) {
callback();
return; let fileIsEtherpad = (fileEnding === ".etherpad");
} let fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm");
let fileIsTXT = (fileEnding === ".txt");
var fileEnding = path.extname(srcFile).toLowerCase(); if (fileIsEtherpad) {
var fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm"); // we do this here so we can see if the pad has quite a few edits
var fileIsTXT = (fileEnding === ".txt"); let _pad = await padManager.getPad(padId);
if (fileIsTXT) useConvertor = false; // Don't use convertor for text files let headCount = _pad.head;
// See https://github.com/ether/etherpad-lite/issues/2572
if (fileIsHTML || (useConvertor === false)) {
// if no convertor only rename
fs.rename(srcFile, destFile, callback);
return;
}
convertor.convertFile(srcFile, destFile, exportExtension, function(err) { if (headCount >= 10) {
//catch convert errors apiLogger.warn("Direct database Import attempt of a pad that already has content, we won't be doing this");
if(err) { throw "padHasData";
console.warn("Converting Error:", err); }
return callback("convertFailed");
}
callback(); const fsp_readFile = util.promisify(fs.readFile);
}); let _text = await fsp_readFile(srcFile, "utf8");
}, req.directDatabaseAccess = true;
await importEtherpad.setPadRaw(padId, _text);
function(callback) { }
if (useConvertor || directDatabaseAccess) {
callback();
return; // convert file to html if necessary
} if (!importHandledByPlugin && !req.directDatabaseAccess) {
if (fileIsTXT) {
// Don't use convertor for text files
useConvertor = false;
}
// Read the file with no encoding for raw buffer access. // See https://github.com/ether/etherpad-lite/issues/2572
fs.readFile(destFile, function(err, buf) { if (fileIsHTML || !useConvertor) {
if (err) throw err; // if no convertor only rename
var isAscii = true; fs.renameSync(srcFile, destFile);
// Check if there are only ascii chars in the uploaded file } else {
for (var i=0, len=buf.length; i<len; i++) { // @TODO - no Promise interface for convertors (yet)
if (buf[i] > 240) { await new Promise((resolve, reject) => {
isAscii=false; convertor.convertFile(srcFile, destFile, exportExtension, function(err) {
break; // catch convert errors
if (err) {
console.warn("Converting Error:", err);
reject("convertFailed");
} }
} resolve();
if (!isAscii) {
callback("uploadFailed");
return;
}
callback();
});
},
//get the pad object
function(callback) {
padManager.getPad(padId, function(err, _pad){
if(ERR(err, callback)) return;
pad = _pad;
callback();
});
},
//read the text
function(callback) {
if (directDatabaseAccess) {
callback();
return;
}
fs.readFile(destFile, "utf8", function(err, _text){
if(ERR(err, callback)) return;
text = _text;
// Title needs to be stripped out else it appends it to the pad..
text = text.replace("<title>", "<!-- <title>");
text = text.replace("</title>","</title>-->");
//node on windows has a delay on releasing of the file lock.
//We add a 100ms delay to work around this
if(os.type().indexOf("Windows") > -1){
setTimeout(function() {callback();}, 100);
} else {
callback();
}
});
},
//change text of the pad and broadcast the changeset
function(callback) {
if(!directDatabaseAccess){
var fileEnding = path.extname(srcFile).toLowerCase();
if (importHandledByPlugin || useConvertor || fileEnding == ".htm" || fileEnding == ".html") {
importHtml.setPadHTML(pad, text, function(e){
if(e) apiLogger.warn("Error importing, possibly caused by malformed HTML");
});
} else {
pad.setText(text);
}
}
// Load the Pad into memory then brodcast updates to all clients
padManager.unloadPad(padId);
padManager.getPad(padId, function(err, _pad){
var pad = _pad;
padManager.unloadPad(padId);
// direct Database Access means a pad user should perform a switchToPad
// and not attempt to recieve updated pad data..
if (directDatabaseAccess) {
callback();
return;
}
padMessageHandler.updatePadClients(pad, function(){
callback();
}); });
}); });
},
//clean up temporary files
function(callback) {
if (directDatabaseAccess) {
callback();
return;
}
try {
fs.unlinkSync(srcFile);
} catch (e) {
console.log(e);
}
try {
fs.unlinkSync(destFile);
} catch (e) {
console.log(e);
}
callback();
} }
], function(err) { }
var status = "ok";
if (!useConvertor && !req.directDatabaseAccess) {
//check for known errors and replace the status // Read the file with no encoding for raw buffer access.
if(err == "uploadFailed" || err == "convertFailed" || err == "padHasData") let buf = await fsp_readFile(destFile);
{
// Check if there are only ascii chars in the uploaded file
let isAscii = ! Array.prototype.some.call(buf, c => (c > 240));
if (!isAscii) {
throw "uploadFailed";
}
}
// get the pad object
let pad = await padManager.getPad(padId);
// read the text
let text;
if (!req.directDatabaseAccess) {
text = await fsp_readFile(destFile, "utf8");
// Title needs to be stripped out else it appends it to the pad..
text = text.replace("<title>", "<!-- <title>");
text = text.replace("</title>","</title>-->");
// node on windows has a delay on releasing of the file lock.
// We add a 100ms delay to work around this
if (os.type().indexOf("Windows") > -1){
await new Promise(resolve => setTimeout(resolve, 100));
}
}
// change text of the pad and broadcast the changeset
if (!req.directDatabaseAccess) {
if (importHandledByPlugin || useConvertor || fileIsHTML) {
try {
importHtml.setPadHTML(pad, text);
} catch (e) {
apiLogger.warn("Error importing, possibly caused by malformed HTML");
}
} else {
pad.setText(text);
}
}
// Load the Pad into memory then broadcast updates to all clients
padManager.unloadPad(padId);
pad = await padManager.getPad(padId);
padManager.unloadPad(padId);
// direct Database Access means a pad user should perform a switchToPad
// and not attempt to receive updated pad data
if (req.directDatabaseAccess) {
return;
}
// tell clients to update
await padMessageHandler.updatePadClients(pad);
// clean up temporary files
/*
* TODO: directly delete the file and handle the eventual error. Checking
* before for existence is prone to race conditions, and does not handle any
* errors anyway.
*/
if (await fsp_exists(srcFile)) {
fsp_unlink(srcFile);
}
if (await fsp_exists(destFile)) {
fsp_unlink(destFile);
}
}
exports.doImport = function (req, res, padId)
{
/**
* NB: abuse the 'req' object by storing an additional
* 'directDatabaseAccess' property on it so that it can
* be passed back in the HTML below.
*
* this is necessary because in the 'throw' paths of
* the function above there's no other way to return
* a value to the caller.
*/
let status = "ok";
doImport(req, res, padId).catch(err => {
// check for known errors and replace the status
if (err == "uploadFailed" || err == "convertFailed" || err == "padHasData") {
status = err; status = err;
err = null; } else {
throw err;
} }
}).then(() => {
ERR(err); // close the connection
//close the connection
res.send( res.send(
"<head> \ "<head> \
<script type='text/javascript' src='../../static/js/jquery.js'></script> \ <script type='text/javascript' src='../../static/js/jquery.js'></script> \
</head> \ </head> \
<script> \ <script> \
$(window).load(function(){ \ $(window).load(function(){ \
var impexp = window.parent.padimpexp.handleFrameCall('" + directDatabaseAccess +"', '" + status + "'); \ var impexp = window.parent.padimpexp.handleFrameCall('" + req.directDatabaseAccess +"', '" + status + "'); \
}) \ }) \
</script>" </script>"
); );
}); });
} }

2075
src/node/handler/PadMessageHandler.js
View file

File diff suppressed because it is too large Load diff

132
src/node/handler/SocketIORouter.js
View file

@ -1,5 +1,5 @@
/** /**
* This is the Socket.IO Router. It routes the Messages between the * This is the Socket.IO Router. It routes the Messages between the
* components of the Server. The components are at the moment: pad and timeslider * components of the Server. The components are at the moment: pad and timeslider
*/ */
@ -19,7 +19,6 @@
* limitations under the License. * limitations under the License.
*/ */
var ERR = require("async-stacktrace");
var log4js = require('log4js'); var log4js = require('log4js');
var messageLogger = log4js.getLogger("message"); var messageLogger = log4js.getLogger("message");
var securityManager = require("../db/SecurityManager"); var securityManager = require("../db/SecurityManager");
@ -31,20 +30,20 @@ var settings = require('../utils/Settings');
* Saves all components * Saves all components
* key is the component name * key is the component name
* value is the component module * value is the component module
*/ */
var components = {}; var components = {};
var socket; var socket;
/** /**
* adds a component * adds a component
*/ */
exports.addComponent = function(moduleName, module) exports.addComponent = function(moduleName, module)
{ {
//save the component // save the component
components[moduleName] = module; components[moduleName] = module;
//give the module the socket // give the module the socket
module.setSocketIO(socket); module.setSocketIO(socket);
} }
@ -52,115 +51,102 @@ exports.addComponent = function(moduleName, module)
* sets the socket.io and adds event functions for routing * sets the socket.io and adds event functions for routing
*/ */
exports.setSocketIO = function(_socket) { exports.setSocketIO = function(_socket) {
//save this socket internaly // save this socket internaly
socket = _socket; socket = _socket;
socket.sockets.on('connection', function(client) socket.sockets.on('connection', function(client)
{ {
// Broken: See http://stackoverflow.com/questions/4647348/send-message-to-specific-client-with-socket-io-and-node-js // Broken: See http://stackoverflow.com/questions/4647348/send-message-to-specific-client-with-socket-io-and-node-js
// Fixed by having a persistant object, ideally this would actually be in the database layer // Fixed by having a persistant object, ideally this would actually be in the database layer
// TODO move to database layer // TODO move to database layer
if(settings.trustProxy && client.handshake.headers['x-forwarded-for'] !== undefined){ if (settings.trustProxy && client.handshake.headers['x-forwarded-for'] !== undefined) {
remoteAddress[client.id] = client.handshake.headers['x-forwarded-for']; remoteAddress[client.id] = client.handshake.headers['x-forwarded-for'];
} } else {
else{
remoteAddress[client.id] = client.handshake.address; remoteAddress[client.id] = client.handshake.address;
} }
var clientAuthorized = false; var clientAuthorized = false;
//wrap the original send function to log the messages // wrap the original send function to log the messages
client._send = client.send; client._send = client.send;
client.send = function(message) { client.send = function(message) {
messageLogger.debug("to " + client.id + ": " + stringifyWithoutPassword(message)); messageLogger.debug("to " + client.id + ": " + stringifyWithoutPassword(message));
client._send(message); client._send(message);
} }
//tell all components about this connect
for(var i in components) {
components[i].handleConnect(client);
}
client.on('message', function(message) // tell all components about this connect
{ for (let i in components) {
if(message.protocolVersion && message.protocolVersion != 2) { components[i].handleConnect(client);
}
client.on('message', async function(message) {
if (message.protocolVersion && message.protocolVersion != 2) {
messageLogger.warn("Protocolversion header is not correct:" + stringifyWithoutPassword(message)); messageLogger.warn("Protocolversion header is not correct:" + stringifyWithoutPassword(message));
return; return;
} }
//client is authorized, everything ok if (clientAuthorized) {
if(clientAuthorized) { // client is authorized, everything ok
handleMessage(client, message); handleMessage(client, message);
} else { //try to authorize the client } else {
if(message.padId !== undefined && message.sessionID !== undefined && message.token !== undefined && message.password !== undefined) { // try to authorize the client
var checkAccessCallback = function(err, statusObject) { if (message.padId !== undefined && message.sessionID !== undefined && message.token !== undefined && message.password !== undefined) {
ERR(err); // check for read-only pads
let padId = message.padId;
//access was granted, mark the client as authorized and handle the message if (padId.indexOf("r.") === 0) {
if(statusObject.accessStatus == "grant") { padId = await readOnlyManager.getPadId(message.padId);
clientAuthorized = true;
handleMessage(client, message);
}
//no access, send the client a message that tell him why
else {
messageLogger.warn("Authentication try failed:" + stringifyWithoutPassword(message));
client.json.send({accessStatus: statusObject.accessStatus});
}
};
if (message.padId.indexOf("r.") === 0) {
readOnlyManager.getPadId(message.padId, function(err, value) {
ERR(err);
securityManager.checkAccess (value, message.sessionID, message.token, message.password, checkAccessCallback);
});
} else {
//this message has everything to try an authorization
securityManager.checkAccess (message.padId, message.sessionID, message.token, message.password, checkAccessCallback);
} }
} else { //drop message
messageLogger.warn("Dropped message cause of bad permissions:" + stringifyWithoutPassword(message)); let { accessStatus } = await securityManager.checkAccess(padId, message.sessionID, message.token, message.password);
if (accessStatus === "grant") {
// access was granted, mark the client as authorized and handle the message
clientAuthorized = true;
handleMessage(client, message);
} else {
// no access, send the client a message that tells him why
messageLogger.warn("Authentication try failed:" + stringifyWithoutPassword(message));
client.json.send({ accessStatus });
}
} else {
// drop message
messageLogger.warn("Dropped message because of bad permissions:" + stringifyWithoutPassword(message));
} }
} }
}); });
client.on('disconnect', function() client.on('disconnect', function() {
{ // tell all components about this disconnect
//tell all components about this disconnect for (let i in components) {
for(var i in components)
{
components[i].handleDisconnect(client); components[i].handleDisconnect(client);
} }
}); });
}); });
} }
//try to handle the message of this client // try to handle the message of this client
function handleMessage(client, message) function handleMessage(client, message)
{ {
if (message.component && components[message.component]) {
if(message.component && components[message.component]) { // check if component is registered in the components array
//check if component is registered in the components array if (components[message.component]) {
if(components[message.component]) {
messageLogger.debug("from " + client.id + ": " + stringifyWithoutPassword(message)); messageLogger.debug("from " + client.id + ": " + stringifyWithoutPassword(message));
components[message.component].handleMessage(client, message); components[message.component].handleMessage(client, message);
} }
} else { } else {
messageLogger.error("Can't route the message:" + stringifyWithoutPassword(message)); messageLogger.error("Can't route the message:" + stringifyWithoutPassword(message));
} }
} }
//returns a stringified representation of a message, removes the password // returns a stringified representation of a message, removes the password
//this ensures there are no passwords in the log // this ensures there are no passwords in the log
function stringifyWithoutPassword(message) function stringifyWithoutPassword(message)
{ {
var newMessage = {}; let newMessage = Object.assign({}, message);
for(var i in message) if (newMessage.password != null) {
{ newMessage.password = "xxx";
if(i == "password" && message[i] != null)
newMessage["password"] = "xxx";
else
newMessage[i]=message[i];
} }
return JSON.stringify(newMessage); return JSON.stringify(newMessage);
} }

124
src/node/hooks/express/adminplugins.js
View file

@ -5,7 +5,7 @@ var plugins = require('ep_etherpad-lite/static/js/pluginfw/plugins');
var _ = require('underscore'); var _ = require('underscore');
var semver = require('semver'); var semver = require('semver');
exports.expressCreateServer = function (hook_name, args, cb) { exports.expressCreateServer = function(hook_name, args, cb) {
args.app.get('/admin/plugins', function(req, res) { args.app.get('/admin/plugins', function(req, res) {
var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins"); var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
var render_args = { var render_args = {
@ -13,91 +13,99 @@ exports.expressCreateServer = function (hook_name, args, cb) {
search_results: {}, search_results: {},
errors: [], errors: [],
}; };
res.send( eejs.require("ep_etherpad-lite/templates/admin/plugins.html", render_args) );
res.send(eejs.require("ep_etherpad-lite/templates/admin/plugins.html", render_args));
}); });
args.app.get('/admin/plugins/info', function(req, res) { args.app.get('/admin/plugins/info', function(req, res) {
var gitCommit = settings.getGitCommit(); var gitCommit = settings.getGitCommit();
var epVersion = settings.getEpVersion(); var epVersion = settings.getEpVersion();
res.send( eejs.require("ep_etherpad-lite/templates/admin/plugins-info.html",
{ res.send(eejs.require("ep_etherpad-lite/templates/admin/plugins-info.html", {
gitCommit: gitCommit, gitCommit: gitCommit,
epVersion: epVersion epVersion: epVersion
}) }));
);
}); });
} }
exports.socketio = function (hook_name, args, cb) { exports.socketio = function(hook_name, args, cb) {
var io = args.io.of("/pluginfw/installer"); var io = args.io.of("/pluginfw/installer");
io.on('connection', function (socket) { io.on('connection', function(socket) {
if (!socket.conn.request.session || !socket.conn.request.session.user || !socket.conn.request.session.user.is_admin) return; if (!socket.conn.request.session || !socket.conn.request.session.user || !socket.conn.request.session.user.is_admin) return;
socket.on("getInstalled", function (query) { socket.on("getInstalled", function(query) {
// send currently installed plugins // send currently installed plugins
var installed = Object.keys(plugins.plugins).map(function(plugin) { var installed = Object.keys(plugins.plugins).map(function(plugin) {
return plugins.plugins[plugin].package return plugins.plugins[plugin].package
}) });
socket.emit("results:installed", {installed: installed}); socket.emit("results:installed", {installed: installed});
}); });
socket.on("checkUpdates", function() { socket.on("checkUpdates", async function() {
// Check plugins for updates // Check plugins for updates
installer.getAvailablePlugins(/*maxCacheAge:*/60*10, function(er, results) { try {
if(er) { let results = await installer.getAvailablePlugins(/*maxCacheAge:*/ 60 * 10);
console.warn(er);
socket.emit("results:updatable", {updatable: {}});
return;
}
var updatable = _(plugins.plugins).keys().filter(function(plugin) { var updatable = _(plugins.plugins).keys().filter(function(plugin) {
if(!results[plugin]) return false; if (!results[plugin]) return false;
var latestVersion = results[plugin].version
var currentVersion = plugins.plugins[plugin].package.version var latestVersion = results[plugin].version;
return semver.gt(latestVersion, currentVersion) var currentVersion = plugins.plugins[plugin].package.version;
return semver.gt(latestVersion, currentVersion);
}); });
socket.emit("results:updatable", {updatable: updatable}); socket.emit("results:updatable", {updatable: updatable});
}); } catch (er) {
}) console.warn(er);
socket.on("getAvailable", function (query) { socket.emit("results:updatable", {updatable: {}});
installer.getAvailablePlugins(/*maxCacheAge:*/false, function (er, results) { }
if(er) {
console.error(er)
results = {}
}
socket.emit("results:available", results);
});
}); });
socket.on("search", function (query) { socket.on("getAvailable", async function(query) {
installer.search(query.searchTerm, /*maxCacheAge:*/60*10, function (er, results) { try {
if(er) { let results = await installer.getAvailablePlugins(/*maxCacheAge:*/ false);
console.error(er) socket.emit("results:available", results);
results = {} } catch (er) {
} console.error(er);
socket.emit("results:available", {});
}
});
socket.on("search", async function(query) {
try {
let results = await installer.search(query.searchTerm, /*maxCacheAge:*/ 60 * 10);
var res = Object.keys(results) var res = Object.keys(results)
.map(function(pluginName) { .map(function(pluginName) {
return results[pluginName] return results[pluginName];
}) })
.filter(function(plugin) { .filter(function(plugin) {
return !plugins.plugins[plugin.name] return !plugins.plugins[plugin.name];
}); });
res = sortPluginList(res, query.sortBy, query.sortDir) res = sortPluginList(res, query.sortBy, query.sortDir)
.slice(query.offset, query.offset+query.limit); .slice(query.offset, query.offset+query.limit);
socket.emit("results:search", {results: res, query: query}); socket.emit("results:search", {results: res, query: query});
}); } catch (er) {
console.error(er);
socket.emit("results:search", {results: {}, query: query});
}
}); });
socket.on("install", function (plugin_name) { socket.on("install", function(plugin_name) {
installer.install(plugin_name, function (er) { installer.install(plugin_name, function(er) {
if(er) console.warn(er) if (er) console.warn(er);
socket.emit("finished:install", {plugin: plugin_name, code: er? er.code : null, error: er? er.message : null}); socket.emit("finished:install", {plugin: plugin_name, code: er? er.code : null, error: er? er.message : null});
}); });
}); });
socket.on("uninstall", function (plugin_name) { socket.on("uninstall", function(plugin_name) {
installer.uninstall(plugin_name, function (er) { installer.uninstall(plugin_name, function(er) {
if(er) console.warn(er) if (er) console.warn(er);
socket.emit("finished:uninstall", {plugin: plugin_name, error: er? er.message : null}); socket.emit("finished:uninstall", {plugin: plugin_name, error: er? er.message : null});
}); });
}); });
@ -106,11 +114,15 @@ exports.socketio = function (hook_name, args, cb) {
function sortPluginList(plugins, property, /*ASC?*/dir) { function sortPluginList(plugins, property, /*ASC?*/dir) {
return plugins.sort(function(a, b) { return plugins.sort(function(a, b) {
if (a[property] < b[property]) if (a[property] < b[property]) {
return dir? -1 : 1; return dir? -1 : 1;
if (a[property] > b[property]) }
return dir? 1 : -1;
if (a[property] > b[property]) {
return dir? 1 : -1;
}
// a must be equal to b // a must be equal to b
return 0; return 0;
}) });
} }

17
src/node/hooks/express/errorhandling.js
View file

@ -11,20 +11,23 @@ exports.gracefulShutdown = function(err) {
console.error(err); console.error(err);
} }
//ensure there is only one graceful shutdown running // ensure there is only one graceful shutdown running
if(exports.onShutdown) return; if (exports.onShutdown) {
return;
}
exports.onShutdown = true; exports.onShutdown = true;
console.log("graceful shutdown..."); console.log("graceful shutdown...");
//do the db shutdown // do the db shutdown
db.db.doShutdown(function() { db.doShutdown().then(function() {
console.log("db sucessfully closed."); console.log("db sucessfully closed.");
process.exit(0); process.exit(0);
}); });
setTimeout(function(){ setTimeout(function() {
process.exit(1); process.exit(1);
}, 3000); }, 3000);
} }
@ -35,14 +38,14 @@ exports.expressCreateServer = function (hook_name, args, cb) {
exports.app = args.app; exports.app = args.app;
// Handle errors // Handle errors
args.app.use(function(err, req, res, next){ args.app.use(function(err, req, res, next) {
// if an error occurs Connect will pass it down // if an error occurs Connect will pass it down
// through these "error-handling" middleware // through these "error-handling" middleware
// allowing you to respond however you like // allowing you to respond however you like
res.status(500).send({ error: 'Sorry, something bad happened!' }); res.status(500).send({ error: 'Sorry, something bad happened!' });
console.error(err.stack? err.stack : err.toString()); console.error(err.stack? err.stack : err.toString());
stats.meter('http500').mark() stats.meter('http500').mark()
}) });
/* /*
* Connect graceful shutdown with sigint and uncaught exception * Connect graceful shutdown with sigint and uncaught exception

43
src/node/hooks/express/importexport.js
View file

@ -5,15 +5,14 @@ var importHandler = require('../../handler/ImportHandler');
var padManager = require("../../db/PadManager"); var padManager = require("../../db/PadManager");
exports.expressCreateServer = function (hook_name, args, cb) { exports.expressCreateServer = function (hook_name, args, cb) {
args.app.get('/p/:pad/:rev?/export/:type', function(req, res, next) { args.app.get('/p/:pad/:rev?/export/:type', async function(req, res, next) {
var types = ["pdf", "doc", "txt", "html", "odt", "etherpad"]; var types = ["pdf", "doc", "txt", "html", "odt", "etherpad"];
//send a 404 if we don't support this filetype //send a 404 if we don't support this filetype
if (types.indexOf(req.params.type) == -1) { if (types.indexOf(req.params.type) == -1) {
next(); return next();
return;
} }
//if abiword is disabled, and this is a format we only support with abiword, output a message // if abiword is disabled, and this is a format we only support with abiword, output a message
if (settings.exportAvailable() == "no" && if (settings.exportAvailable() == "no" &&
["odt", "pdf", "doc"].indexOf(req.params.type) !== -1) { ["odt", "pdf", "doc"].indexOf(req.params.type) !== -1) {
res.send("This export is not enabled at this Etherpad instance. Set the path to Abiword or SOffice in settings.json to enable this feature"); res.send("This export is not enabled at this Etherpad instance. Set the path to Abiword or SOffice in settings.json to enable this feature");
@ -22,30 +21,26 @@ exports.expressCreateServer = function (hook_name, args, cb) {
res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Origin", "*");
hasPadAccess(req, res, function() { if (await hasPadAccess(req, res)) {
console.log('req.params.pad', req.params.pad); console.log('req.params.pad', req.params.pad);
padManager.doesPadExists(req.params.pad, function(err, exists) let exists = await padManager.doesPadExists(req.params.pad);
{ if (!exists) {
if(!exists) { return next();
return next(); }
}
exportHandler.doExport(req, res, req.params.pad, req.params.type); exportHandler.doExport(req, res, req.params.pad, req.params.type);
}); }
});
}); });
//handle import requests // handle import requests
args.app.post('/p/:pad/import', function(req, res, next) { args.app.post('/p/:pad/import', async function(req, res, next) {
hasPadAccess(req, res, function() { if (await hasPadAccess(req, res)) {
padManager.doesPadExists(req.params.pad, function(err, exists) let exists = await padManager.doesPadExists(req.params.pad);
{ if (!exists) {
if(!exists) { return next();
return next(); }
}
importHandler.doImport(req, res, req.params.pad); importHandler.doImport(req, res, req.params.pad);
}); }
});
}); });
} }

68
src/node/hooks/express/padreadonly.js
View file

@ -1,64 +1,26 @@
var async = require('async');
var ERR = require("async-stacktrace");
var readOnlyManager = require("../../db/ReadOnlyManager"); var readOnlyManager = require("../../db/ReadOnlyManager");
var hasPadAccess = require("../../padaccess"); var hasPadAccess = require("../../padaccess");
var exporthtml = require("../../utils/ExportHtml"); var exporthtml = require("../../utils/ExportHtml");
exports.expressCreateServer = function (hook_name, args, cb) { exports.expressCreateServer = function (hook_name, args, cb) {
//serve read only pad // serve read only pad
args.app.get('/ro/:id', function(req, res) args.app.get('/ro/:id', async function(req, res) {
{
var html;
var padId;
async.series([ // translate the read only pad to a padId
//translate the read only pad to a padId let padId = await readOnlyManager.getPadId(req.params.id);
function(callback) if (padId == null) {
{ res.status(404).send('404 - Not Found');
readOnlyManager.getPadId(req.params.id, function(err, _padId) return;
{ }
if(ERR(err, callback)) return;
padId = _padId; // we need that to tell hasPadAcess about the pad
req.params.pad = padId;
//we need that to tell hasPadAcess about the pad if (await hasPadAccess(req, res)) {
req.params.pad = padId; // render the html document
html = await exporthtml.getPadHTMLDocument(padId, null);
callback(); res.send(html);
}); }
},
//render the html document
function(callback)
{
//return if the there is no padId
if(padId == null)
{
callback("notfound");
return;
}
hasPadAccess(req, res, function()
{
//render the html document
exporthtml.getPadHTMLDocument(padId, null, function(err, _html)
{
if(ERR(err, callback)) return;
html = _html;
callback();
});
});
}
], function(err)
{
//throw any unexpected error
if(err && err != "notfound")
ERR(err);
if(err == "notfound")
res.status(404).send('404 - Not Found');
else
res.send(html);
});
}); });
} }

41
src/node/hooks/express/padurlsanitize.js
View file

@ -2,31 +2,28 @@ var padManager = require('../../db/PadManager');
var url = require('url'); var url = require('url');
exports.expressCreateServer = function (hook_name, args, cb) { exports.expressCreateServer = function (hook_name, args, cb) {
//redirects browser to the pad's sanitized url if needed. otherwise, renders the html
args.app.param('pad', function (req, res, next, padId) { // redirects browser to the pad's sanitized url if needed. otherwise, renders the html
//ensure the padname is valid and the url doesn't end with a / args.app.param('pad', async function (req, res, next, padId) {
if(!padManager.isValidPadId(padId) || /\/$/.test(req.url)) // ensure the padname is valid and the url doesn't end with a /
{ if (!padManager.isValidPadId(padId) || /\/$/.test(req.url)) {
res.status(404).send('Such a padname is forbidden'); res.status(404).send('Such a padname is forbidden');
return; return;
} }
padManager.sanitizePadId(padId, function(sanitizedPadId) { let sanitizedPadId = await padManager.sanitizePadId(padId);
//the pad id was sanitized, so we redirect to the sanitized version
if(sanitizedPadId != padId) if (sanitizedPadId === padId) {
{ // the pad id was fine, so just render it
var real_url = sanitizedPadId; next();
real_url = encodeURIComponent(real_url); } else {
var query = url.parse(req.url).query; // the pad id was sanitized, so we redirect to the sanitized version
if ( query ) real_url += '?' + query; var real_url = sanitizedPadId;
res.header('Location', real_url); real_url = encodeURIComponent(real_url);
res.status(302).send('You should be redirected to <a href="' + real_url + '">' + real_url + '</a>'); var query = url.parse(req.url).query;
} if ( query ) real_url += '?' + query;
//the pad id was fine, so just render it res.header('Location', real_url);
else res.status(302).send('You should be redirected to <a href="' + real_url + '">' + real_url + '</a>');
{ }
next();
}
});
}); });
} }

92
src/node/hooks/express/tests.js
View file

@ -1,40 +1,33 @@
var path = require("path") var path = require("path")
, npm = require("npm") , npm = require("npm")
, fs = require("fs") , fs = require("fs")
, async = require("async"); , util = require("util");
exports.expressCreateServer = function (hook_name, args, cb) { exports.expressCreateServer = function (hook_name, args, cb) {
args.app.get('/tests/frontend/specs_list.js', function(req, res){ args.app.get('/tests/frontend/specs_list.js', async function(req, res) {
let [coreTests, pluginTests] = await Promise.all([
async.parallel({ exports.getCoreTests(),
coreSpecs: function(callback){ exports.getPluginTests()
exports.getCoreTests(callback); ]);
},
pluginSpecs: function(callback){
exports.getPluginTests(callback);
}
},
function(err, results){
var files = results.coreSpecs; // push the core specs to a file object
files = files.concat(results.pluginSpecs); // add the plugin Specs to the core specs
console.debug("Sent browser the following test specs:", files.sort());
res.send("var specs_list = " + JSON.stringify(files.sort()) + ";\n");
});
// merge the two sets of results
let files = [].concat(coreTests, pluginTests).sort();
console.debug("Sent browser the following test specs:", files);
res.send("var specs_list = " + JSON.stringify(files) + ";\n");
}); });
// path.join seems to normalize by default, but we'll just be explicit // path.join seems to normalize by default, but we'll just be explicit
var rootTestFolder = path.normalize(path.join(npm.root, "../tests/frontend/")); var rootTestFolder = path.normalize(path.join(npm.root, "../tests/frontend/"));
var url2FilePath = function(url){ var url2FilePath = function(url) {
var subPath = url.substr("/tests/frontend".length); var subPath = url.substr("/tests/frontend".length);
if (subPath == ""){ if (subPath == "") {
subPath = "index.html" subPath = "index.html"
} }
subPath = subPath.split("?")[0]; subPath = subPath.split("?")[0];
var filePath = path.normalize(path.join(rootTestFolder, subPath)); var filePath = path.normalize(path.join(rootTestFolder, subPath));
// make sure we jail the paths to the test folder, otherwise serve index // make sure we jail the paths to the test folder, otherwise serve index
if (filePath.indexOf(rootTestFolder) !== 0) { if (filePath.indexOf(rootTestFolder) !== 0) {
filePath = path.join(rootTestFolder, "index.html"); filePath = path.join(rootTestFolder, "index.html");
@ -46,13 +39,13 @@ exports.expressCreateServer = function (hook_name, args, cb) {
var specFilePath = url2FilePath(req.url); var specFilePath = url2FilePath(req.url);
var specFileName = path.basename(specFilePath); var specFileName = path.basename(specFilePath);
fs.readFile(specFilePath, function(err, content){ fs.readFile(specFilePath, function(err, content) {
if(err){ return res.send(500); } if (err) { return res.send(500); }
content = "describe(" + JSON.stringify(specFileName) + ", function(){ " + content + " });"; content = "describe(" + JSON.stringify(specFileName) + ", function(){ " + content + " });";
res.send(content); res.send(content);
}); });
}); });
args.app.get('/tests/frontend/*', function (req, res) { args.app.get('/tests/frontend/*', function (req, res) {
@ -62,30 +55,33 @@ exports.expressCreateServer = function (hook_name, args, cb) {
args.app.get('/tests/frontend', function (req, res) { args.app.get('/tests/frontend', function (req, res) {
res.redirect('/tests/frontend/'); res.redirect('/tests/frontend/');
});
}
exports.getPluginTests = function(callback){
var pluginSpecs = [];
var plugins = fs.readdirSync('node_modules');
plugins.forEach(function(plugin){
if(fs.existsSync("node_modules/"+plugin+"/static/tests/frontend/specs")){ // if plugins exists
var specFiles = fs.readdirSync("node_modules/"+plugin+"/static/tests/frontend/specs/");
async.forEach(specFiles, function(spec){ // for each specFile push it to pluginSpecs
pluginSpecs.push("/static/plugins/"+plugin+"/static/tests/frontend/specs/" + spec);
},
function(err){
// blow up if something bad happens!
});
}
});
callback(null, pluginSpecs);
}
exports.getCoreTests = function(callback){
fs.readdir('tests/frontend/specs', function(err, coreSpecs){ // get the core test specs
if(err){ return res.send(500); }
callback(null, coreSpecs);
}); });
} }
const readdir = util.promisify(fs.readdir);
exports.getPluginTests = async function(callback) {
const moduleDir = "node_modules/";
const specPath = "/static/tests/frontend/specs/";
const staticDir = "/static/plugins/";
let pluginSpecs = [];
let plugins = await readdir(moduleDir);
let promises = plugins
.map(plugin => [ plugin, moduleDir + plugin + specPath] )
.filter(([plugin, specDir]) => fs.existsSync(specDir)) // check plugin exists
.map(([plugin, specDir]) => {
return readdir(specDir)
.then(specFiles => specFiles.map(spec => {
pluginSpecs.push(staticDir + plugin + specPath + spec);
}));
});
return Promise.all(promises).then(() => pluginSpecs);
}
exports.getCoreTests = function() {
// get the core test specs
return readdir('tests/frontend/specs');
}

23
src/node/padaccess.js
View file

@ -1,17 +1,20 @@
var ERR = require("async-stacktrace");
var securityManager = require('./db/SecurityManager'); var securityManager = require('./db/SecurityManager');
//checks for padAccess // checks for padAccess
module.exports = function (req, res, callback) { module.exports = async function (req, res) {
securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password, function(err, accessObj) { try {
if(ERR(err, callback)) return; let accessObj = await securityManager.checkAccess(req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password);
//there is access, continue if (accessObj.accessStatus === "grant") {
if(accessObj.accessStatus == "grant") { // there is access, continue
callback(); return true;
//no access
} else { } else {
// no access
res.status(403).send("403 - Can't touch this"); res.status(403).send("403 - Can't touch this");
return false;
} }
}); } catch (err) {
// @TODO - send internal server error here?
throw err;
}
} }

88
src/node/server.js
View file

@ -1,7 +1,7 @@
#!/usr/bin/env node #!/usr/bin/env node
/** /**
* This module is started with bin/run.sh. It sets up a Express HTTP and a Socket.IO Server. * This module is started with bin/run.sh. It sets up a Express HTTP and a Socket.IO Server.
* Static file Requests are answered directly from this module, Socket.IO messages are passed * Static file Requests are answered directly from this module, Socket.IO messages are passed
* to MessageHandler and minfied requests are passed to minified. * to MessageHandler and minfied requests are passed to minified.
*/ */
@ -22,7 +22,6 @@
*/ */
var log4js = require('log4js') var log4js = require('log4js')
, async = require('async')
, NodeVersion = require('./utils/NodeVersion') , NodeVersion = require('./utils/NodeVersion')
; ;
@ -46,57 +45,40 @@ NodeVersion.enforceMinNodeVersion('8.9.0');
*/ */
var stats = require('./stats'); var stats = require('./stats');
stats.gauge('memoryUsage', function() { stats.gauge('memoryUsage', function() {
return process.memoryUsage().rss return process.memoryUsage().rss;
}) });
var settings /*
, db * no use of let or await here because it would cause startup
, plugins * to fail completely on very early versions of NodeJS
, hooks; */
var npm = require("npm/lib/npm.js"); var npm = require("npm/lib/npm.js");
async.waterfall([ npm.load({}, function() {
// load npm var settings = require('./utils/Settings');
function(callback) { var db = require('./db/DB');
npm.load({}, function(er) { var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
callback(er) var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks");
hooks.plugins = plugins;
db.init()
.then(plugins.update)
.then(function() {
console.info("Installed plugins: " + plugins.formatPluginsWithVersion());
console.debug("Installed parts:\n" + plugins.formatParts());
console.debug("Installed hooks:\n" + plugins.formatHooks());
// Call loadSettings hook
hooks.aCallAll("loadSettings", { settings: settings });
// initalize the http server
hooks.callAll("createServer", {});
}) })
}, .catch(function(e) {
console.error("exception thrown: " + e.message);
// load everything if (e.stack) {
function(callback) { console.log(e.stack);
settings = require('./utils/Settings'); }
db = require('./db/DB'); process.exit(1);
plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins"); });
hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks"); });
hooks.plugins = plugins;
callback();
},
//initalize the database
function (callback)
{
db.init(callback);
},
function(callback) {
plugins.update(callback)
},
function (callback) {
console.info("Installed plugins: " + plugins.formatPluginsWithVersion());
console.debug("Installed parts:\n" + plugins.formatParts());
console.debug("Installed hooks:\n" + plugins.formatHooks());
// Call loadSettings hook
hooks.aCallAll("loadSettings", { settings: settings });
callback();
},
//initalize the http server
function (callback)
{
hooks.callAll("createServer", {});
callback(null);
}
]);

90
src/node/utils/ExportEtherpad.js
View file

@ -15,58 +15,48 @@
*/ */
var async = require("async"); let db = require("../db/DB");
var db = require("../db/DB").db;
var ERR = require("async-stacktrace");
exports.getPadRaw = function(padId, callback){ exports.getPadRaw = async function(padId) {
async.waterfall([
function(cb){
db.get("pad:"+padId, cb);
},
function(padcontent,cb){
var records = ["pad:"+padId];
for (var i = 0; i <= padcontent.head; i++) {
records.push("pad:"+padId+":revs:" + i);
}
for (var i = 0; i <= padcontent.chatHead; i++) { let padKey = "pad:" + padId;
records.push("pad:"+padId+":chat:" + i); let padcontent = await db.get(padKey);
}
var data = {}; let records = [ padKey ];
for (let i = 0; i <= padcontent.head; i++) {
async.forEachSeries(Object.keys(records), function(key, r){ records.push(padKey + ":revs:" + i);
// For each piece of info about a pad.
db.get(records[key], function(err, entry){
data[records[key]] = entry;
// Get the Pad Authors
if(entry.pool && entry.pool.numToAttrib){
var authors = entry.pool.numToAttrib;
async.forEachSeries(Object.keys(authors), function(k, c){
if(authors[k][0] === "author"){
var authorId = authors[k][1];
// Get the author info
db.get("globalAuthor:"+authorId, function(e, authorEntry){
if(authorEntry && authorEntry.padIDs) authorEntry.padIDs = padId;
if(!e) data["globalAuthor:"+authorId] = authorEntry;
});
}
// console.log("authorsK", authors[k]);
c(null);
});
}
r(null); // callback;
});
}, function(err){
cb(err, data);
})
} }
], function(err, data){
callback(null, data); for (let i = 0; i <= padcontent.chatHead; i++) {
}); records.push(padKey + ":chat:" + i);
}
let data = {};
for (let key of records) {
// For each piece of info about a pad.
let entry = data[key] = await db.get(key);
// Get the Pad Authors
if (entry.pool && entry.pool.numToAttrib) {
let authors = entry.pool.numToAttrib;
for (let k of Object.keys(authors)) {
if (authors[k][0] === "author") {
let authorId = authors[k][1];
// Get the author info
let authorEntry = await db.get("globalAuthor:" + authorId);
if (authorEntry) {
data["globalAuthor:" + authorId] = authorEntry;
if (authorEntry.padIDs) {
authorEntry.padIDs = padId;
}
}
}
}
}
}
return data;
} }

91
src/node/utils/ExportHtml.js
View file

@ -14,11 +14,8 @@
* limitations under the License. * limitations under the License.
*/ */
var async = require("async");
var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var padManager = require("../db/PadManager"); var padManager = require("../db/PadManager");
var ERR = require("async-stacktrace");
var _ = require('underscore'); var _ = require('underscore');
var Security = require('ep_etherpad-lite/static/js/security'); var Security = require('ep_etherpad-lite/static/js/security');
var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks');
@ -26,45 +23,17 @@ var eejs = require('ep_etherpad-lite/node/eejs');
var _analyzeLine = require('./ExportHelper')._analyzeLine; var _analyzeLine = require('./ExportHelper')._analyzeLine;
var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace; var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace;
function getPadHTML(pad, revNum, callback) async function getPadHTML(pad, revNum)
{ {
var atext = pad.atext; let atext = pad.atext;
var html;
async.waterfall([
// fetch revision atext // fetch revision atext
function (callback) if (revNum != undefined) {
{ atext = await pad.getInternalRevisionAText(revNum);
if (revNum != undefined) }
{
pad.getInternalRevisionAText(revNum, function (err, revisionAtext)
{
if(ERR(err, callback)) return;
atext = revisionAtext;
callback();
});
}
else
{
callback(null);
}
},
// convert atext to html // convert atext to html
return getHTMLFromAtext(pad, atext);
function (callback)
{
html = getHTMLFromAtext(pad, atext);
callback(null);
}],
// run final callback
function (err)
{
if(ERR(err, callback)) return;
callback(null, html);
});
} }
exports.getPadHTML = getPadHTML; exports.getPadHTML = getPadHTML;
@ -81,15 +50,16 @@ function getHTMLFromAtext(pad, atext, authorColors)
// prepare tags stored as ['tag', true] to be exported // prepare tags stored as ['tag', true] to be exported
hooks.aCallAll("exportHtmlAdditionalTags", pad, function(err, newProps){ hooks.aCallAll("exportHtmlAdditionalTags", pad, function(err, newProps){
newProps.forEach(function (propName, i){ newProps.forEach(function (propName, i) {
tags.push(propName); tags.push(propName);
props.push(propName); props.push(propName);
}); });
}); });
// prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML // prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML
// with tags like <span data-tag="value"> // with tags like <span data-tag="value">
hooks.aCallAll("exportHtmlAdditionalTagsWithData", pad, function(err, newProps){ hooks.aCallAll("exportHtmlAdditionalTagsWithData", pad, function(err, newProps){
newProps.forEach(function (propName, i){ newProps.forEach(function (propName, i) {
tags.push('span data-' + propName[0] + '="' + propName[1] + '"'); tags.push('span data-' + propName[0] + '="' + propName[1] + '"');
props.push(propName); props.push(propName);
}); });
@ -453,38 +423,31 @@ function getHTMLFromAtext(pad, atext, authorColors)
hooks.aCallAll("getLineHTMLForExport", context); hooks.aCallAll("getLineHTMLForExport", context);
pieces.push(context.lineContent, "<br>"); pieces.push(context.lineContent, "<br>");
}
} }
}
return pieces.join(''); return pieces.join('');
} }
exports.getPadHTMLDocument = function (padId, revNum, callback) exports.getPadHTMLDocument = async function (padId, revNum)
{ {
padManager.getPad(padId, function (err, pad) let pad = await padManager.getPad(padId);
{
if(ERR(err, callback)) return;
var stylesForExportCSS = ""; // Include some Styles into the Head for Export
// Include some Styles into the Head for Export let stylesForExportCSS = "";
hooks.aCallAll("stylesForExport", padId, function(err, stylesForExport){ let stylesForExport = await hooks.aCallAll("stylesForExport", padId);
stylesForExport.forEach(function(css){ stylesForExport.forEach(function(css){
stylesForExportCSS += css; stylesForExportCSS += css;
});
getPadHTML(pad, revNum, function (err, html)
{
if(ERR(err, callback)) return;
var exportedDoc = eejs.require("ep_etherpad-lite/templates/export_html.html", {
body: html,
padId: Security.escapeHTML(padId),
extraCSS: stylesForExportCSS
});
callback(null, exportedDoc);
});
});
}); });
};
let html = await getPadHTML(pad, revNum);
return eejs.require("ep_etherpad-lite/templates/export_html.html", {
body: html,
padId: Security.escapeHTML(padId),
extraCSS: stylesForExportCSS
});
}
// copied from ACE // copied from ACE
var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/; var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/;

182
src/node/utils/ExportTxt.js
View file

@ -18,54 +18,22 @@
* limitations under the License. * limitations under the License.
*/ */
var async = require("async");
var Changeset = require("ep_etherpad-lite/static/js/Changeset"); var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var padManager = require("../db/PadManager"); var padManager = require("../db/PadManager");
var ERR = require("async-stacktrace");
var _analyzeLine = require('./ExportHelper')._analyzeLine; var _analyzeLine = require('./ExportHelper')._analyzeLine;
// This is slightly different than the HTML method as it passes the output to getTXTFromAText // This is slightly different than the HTML method as it passes the output to getTXTFromAText
function getPadTXT(pad, revNum, callback) var getPadTXT = async function(pad, revNum)
{ {
var atext = pad.atext; let atext = pad.atext;
var html;
async.waterfall([
// fetch revision atext
if (revNum != undefined) {
function (callback) // fetch revision atext
{ atext = await pad.getInternalRevisionAText(revNum);
if (revNum != undefined) }
{
pad.getInternalRevisionAText(revNum, function (err, revisionAtext)
{
if(ERR(err, callback)) return;
atext = revisionAtext;
callback();
});
}
else
{
callback(null);
}
},
// convert atext to html // convert atext to html
return getTXTFromAtext(pad, atext);
function (callback)
{
html = getTXTFromAtext(pad, atext); // only this line is different to the HTML function
callback(null);
}],
// run final callback
function (err)
{
if(ERR(err, callback)) return;
callback(null, html);
});
} }
// This is different than the functionality provided in ExportHtml as it provides formatting // This is different than the functionality provided in ExportHtml as it provides formatting
@ -80,17 +48,14 @@ function getTXTFromAtext(pad, atext, authorColors)
var anumMap = {}; var anumMap = {};
var css = ""; var css = "";
props.forEach(function (propName, i) props.forEach(function(propName, i) {
{
var propTrueNum = apool.putAttrib([propName, true], true); var propTrueNum = apool.putAttrib([propName, true], true);
if (propTrueNum >= 0) if (propTrueNum >= 0) {
{
anumMap[propTrueNum] = i; anumMap[propTrueNum] = i;
} }
}); });
function getLineTXT(text, attribs) function getLineTXT(text, attribs) {
{
var propVals = [false, false, false]; var propVals = [false, false, false];
var ENTER = 1; var ENTER = 1;
var STAY = 2; var STAY = 2;
@ -106,94 +71,77 @@ function getTXTFromAtext(pad, atext, authorColors)
var idx = 0; var idx = 0;
function processNextChars(numChars) function processNextChars(numChars) {
{ if (numChars <= 0) {
if (numChars <= 0)
{
return; return;
} }
var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars)); var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars));
idx += numChars; idx += numChars;
while (iter.hasNext()) while (iter.hasNext()) {
{
var o = iter.next(); var o = iter.next();
var propChanged = false; var propChanged = false;
Changeset.eachAttribNumber(o.attribs, function (a)
{ Changeset.eachAttribNumber(o.attribs, function(a) {
if (a in anumMap) if (a in anumMap) {
{
var i = anumMap[a]; // i = 0 => bold, etc. var i = anumMap[a]; // i = 0 => bold, etc.
if (!propVals[i])
{ if (!propVals[i]) {
propVals[i] = ENTER; propVals[i] = ENTER;
propChanged = true; propChanged = true;
} } else {
else
{
propVals[i] = STAY; propVals[i] = STAY;
} }
} }
}); });
for (var i = 0; i < propVals.length; i++)
{ for (var i = 0; i < propVals.length; i++) {
if (propVals[i] === true) if (propVals[i] === true) {
{
propVals[i] = LEAVE; propVals[i] = LEAVE;
propChanged = true; propChanged = true;
} } else if (propVals[i] === STAY) {
else if (propVals[i] === STAY) // set it back
{ propVals[i] = true;
propVals[i] = true; // set it back
} }
} }
// now each member of propVal is in {false,LEAVE,ENTER,true} // now each member of propVal is in {false,LEAVE,ENTER,true}
// according to what happens at start of span // according to what happens at start of span
if (propChanged) if (propChanged) {
{
// leaving bold (e.g.) also leaves italics, etc. // leaving bold (e.g.) also leaves italics, etc.
var left = false; var left = false;
for (var i = 0; i < propVals.length; i++)
{ for (var i = 0; i < propVals.length; i++) {
var v = propVals[i]; var v = propVals[i];
if (!left)
{ if (!left) {
if (v === LEAVE) if (v === LEAVE) {
{
left = true; left = true;
} }
} } else {
else if (v === true) {
{ // tag will be closed and re-opened
if (v === true) propVals[i] = STAY;
{
propVals[i] = STAY; // tag will be closed and re-opened
} }
} }
} }
var tags2close = []; var tags2close = [];
for (var i = propVals.length - 1; i >= 0; i--) for (var i = propVals.length - 1; i >= 0; i--) {
{ if (propVals[i] === LEAVE) {
if (propVals[i] === LEAVE)
{
//emitCloseTag(i); //emitCloseTag(i);
tags2close.push(i); tags2close.push(i);
propVals[i] = false; propVals[i] = false;
} } else if (propVals[i] === STAY) {
else if (propVals[i] === STAY)
{
//emitCloseTag(i); //emitCloseTag(i);
tags2close.push(i); tags2close.push(i);
} }
} }
for (var i = 0; i < propVals.length; i++) for (var i = 0; i < propVals.length; i++) {
{ if (propVals[i] === ENTER || propVals[i] === STAY) {
if (propVals[i] === ENTER || propVals[i] === STAY)
{
propVals[i] = true; propVals[i] = true;
} }
} }
@ -201,9 +149,9 @@ function getTXTFromAtext(pad, atext, authorColors)
} // end if (propChanged) } // end if (propChanged)
var chars = o.chars; var chars = o.chars;
if (o.lines) if (o.lines) {
{ // exclude newline at end of line, if present
chars--; // exclude newline at end of line, if present chars--;
} }
var s = taker.take(chars); var s = taker.take(chars);
@ -220,19 +168,19 @@ function getTXTFromAtext(pad, atext, authorColors)
} // end iteration over spans in line } // end iteration over spans in line
var tags2close = []; var tags2close = [];
for (var i = propVals.length - 1; i >= 0; i--) for (var i = propVals.length - 1; i >= 0; i--) {
{ if (propVals[i]) {
if (propVals[i])
{
tags2close.push(i); tags2close.push(i);
propVals[i] = false; propVals[i] = false;
} }
} }
} // end processNextChars } // end processNextChars
processNextChars(text.length - idx); processNextChars(text.length - idx);
return(assem.toString()); return(assem.toString());
} // end getLineHTML } // end getLineHTML
var pieces = [css]; var pieces = [css];
// Need to deal with constraints imposed on HTML lists; can // Need to deal with constraints imposed on HTML lists; can
@ -242,42 +190,38 @@ function getTXTFromAtext(pad, atext, authorColors)
// so we want to do something reasonable there. We also // so we want to do something reasonable there. We also
// want to deal gracefully with blank lines. // want to deal gracefully with blank lines.
// => keeps track of the parents level of indentation // => keeps track of the parents level of indentation
for (var i = 0; i < textLines.length; i++) for (var i = 0; i < textLines.length; i++) {
{
var line = _analyzeLine(textLines[i], attribLines[i], apool); var line = _analyzeLine(textLines[i], attribLines[i], apool);
var lineContent = getLineTXT(line.text, line.aline); var lineContent = getLineTXT(line.text, line.aline);
if(line.listTypeName == "bullet"){
if (line.listTypeName == "bullet") {
lineContent = "* " + lineContent; // add a bullet lineContent = "* " + lineContent; // add a bullet
} }
if(line.listLevel > 0){
for (var j = line.listLevel - 1; j >= 0; j--){ if (line.listLevel > 0) {
for (var j = line.listLevel - 1; j >= 0; j--) {
pieces.push('\t'); pieces.push('\t');
} }
if(line.listTypeName == "number"){
if (line.listTypeName == "number") {
pieces.push(line.listLevel + ". "); pieces.push(line.listLevel + ". ");
// This is bad because it doesn't truly reflect what the user // This is bad because it doesn't truly reflect what the user
// sees because browsers do magic on nested <ol><li>s // sees because browsers do magic on nested <ol><li>s
} }
pieces.push(lineContent, '\n'); pieces.push(lineContent, '\n');
}else{ } else {
pieces.push(lineContent, '\n'); pieces.push(lineContent, '\n');
} }
} }
return pieces.join(''); return pieces.join('');
} }
exports.getTXTFromAtext = getTXTFromAtext; exports.getTXTFromAtext = getTXTFromAtext;
exports.getPadTXTDocument = function (padId, revNum, callback) exports.getPadTXTDocument = async function(padId, revNum)
{ {
padManager.getPad(padId, function (err, pad) let pad = await padManager.getPad(padId);
{ return getPadTXT(pad, revNum);
if(ERR(err, callback)) return; }
getPadTXT(pad, revNum, function (err, html)
{
if(ERR(err, callback)) return;
callback(null, html);
});
});
};

70
src/node/utils/ImportEtherpad.js
View file

@ -15,60 +15,56 @@
*/ */
var log4js = require('log4js'); var log4js = require('log4js');
var async = require("async"); const db = require("../db/DB");
var db = require("../db/DB").db;
exports.setPadRaw = function(padId, records, callback){ exports.setPadRaw = function(padId, records)
{
records = JSON.parse(records); records = JSON.parse(records);
async.eachSeries(Object.keys(records), function(key, cb){ Object.keys(records).forEach(async function(key) {
var value = records[key] let value = records[key];
if(!value){ if (!value) {
return setImmediate(cb); return;
} }
// Author data let newKey;
if(value.padIDs){
// rewrite author pad ids if (value.padIDs) {
// Author data - rewrite author pad ids
value.padIDs[padId] = 1; value.padIDs[padId] = 1;
var newKey = key; newKey = key;
// Does this author already exist? // Does this author already exist?
db.get(key, function(err, author){ let author = await db.get(key);
if(author){
// Yes, add the padID to the author.. if (author) {
if( Object.prototype.toString.call(author) === '[object Array]'){ // Yes, add the padID to the author
author.padIDs.push(padId); if (Object.prototype.toString.call(author) === '[object Array]') {
} author.padIDs.push(padId);
value = author;
}else{
// No, create a new array with the author info in
value.padIDs = [padId];
} }
});
// Not author data, probably pad data value = author;
}else{ } else {
// we can split it to look to see if its pad data // No, create a new array with the author info in
var oldPadId = key.split(":"); value.padIDs = [ padId ];
}
// we know its pad data.. } else {
if(oldPadId[0] === "pad"){ // Not author data, probably pad data
// we can split it to look to see if it's pad data
let oldPadId = key.split(":");
// we know it's pad data
if (oldPadId[0] === "pad") {
// so set the new pad id for the author // so set the new pad id for the author
oldPadId[1] = padId; oldPadId[1] = padId;
// and create the value // and create the value
var newKey = oldPadId.join(":"); // create the new key newKey = oldPadId.join(":"); // create the new key
} }
} }
// Write the value to the server
db.set(newKey, value);
setImmediate(cb); // Write the value to the server
}, function(){ await db.set(newKey, value);
callback(null, true);
}); });
} }

31
src/node/utils/ImportHtml.js
View file

@ -19,7 +19,7 @@ var Changeset = require("ep_etherpad-lite/static/js/Changeset");
var contentcollector = require("ep_etherpad-lite/static/js/contentcollector"); var contentcollector = require("ep_etherpad-lite/static/js/contentcollector");
var cheerio = require("cheerio"); var cheerio = require("cheerio");
function setPadHTML(pad, html, callback) exports.setPadHTML = function(pad, html)
{ {
var apiLogger = log4js.getLogger("ImportHtml"); var apiLogger = log4js.getLogger("ImportHtml");
@ -36,19 +36,22 @@ function setPadHTML(pad, html, callback)
// Convert a dom tree into a list of lines and attribute liens // Convert a dom tree into a list of lines and attribute liens
// using the content collector object // using the content collector object
var cc = contentcollector.makeContentCollector(true, null, pad.pool); var cc = contentcollector.makeContentCollector(true, null, pad.pool);
try{ // we use a try here because if the HTML is bad it will blow up try {
// we use a try here because if the HTML is bad it will blow up
cc.collectContent(doc); cc.collectContent(doc);
}catch(e){ } catch(e) {
apiLogger.warn("HTML was not properly formed", e); apiLogger.warn("HTML was not properly formed", e);
return callback(e); // We don't process the HTML because it was bad..
// don't process the HTML because it was bad
throw e;
} }
var result = cc.finish(); var result = cc.finish();
apiLogger.debug('Lines:'); apiLogger.debug('Lines:');
var i; var i;
for (i = 0; i < result.lines.length; i += 1) for (i = 0; i < result.lines.length; i++) {
{
apiLogger.debug('Line ' + (i + 1) + ' text: ' + result.lines[i]); apiLogger.debug('Line ' + (i + 1) + ' text: ' + result.lines[i]);
apiLogger.debug('Line ' + (i + 1) + ' attributes: ' + result.lineAttribs[i]); apiLogger.debug('Line ' + (i + 1) + ' attributes: ' + result.lineAttribs[i]);
} }
@ -59,18 +62,15 @@ function setPadHTML(pad, html, callback)
apiLogger.debug(newText); apiLogger.debug(newText);
var newAttribs = result.lineAttribs.join('|1+1') + '|1+1'; var newAttribs = result.lineAttribs.join('|1+1') + '|1+1';
function eachAttribRun(attribs, func /*(startInNewText, endInNewText, attribs)*/ ) function eachAttribRun(attribs, func /*(startInNewText, endInNewText, attribs)*/ ) {
{
var attribsIter = Changeset.opIterator(attribs); var attribsIter = Changeset.opIterator(attribs);
var textIndex = 0; var textIndex = 0;
var newTextStart = 0; var newTextStart = 0;
var newTextEnd = newText.length; var newTextEnd = newText.length;
while (attribsIter.hasNext()) while (attribsIter.hasNext()) {
{
var op = attribsIter.next(); var op = attribsIter.next();
var nextIndex = textIndex + op.chars; var nextIndex = textIndex + op.chars;
if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) if (!(nextIndex <= newTextStart || textIndex >= newTextEnd)) {
{
func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs); func(Math.max(newTextStart, textIndex), Math.min(newTextEnd, nextIndex), op.attribs);
} }
textIndex = nextIndex; textIndex = nextIndex;
@ -81,17 +81,14 @@ function setPadHTML(pad, html, callback)
var builder = Changeset.builder(1); var builder = Changeset.builder(1);
// assemble each line into the builder // assemble each line into the builder
eachAttribRun(newAttribs, function(start, end, attribs) eachAttribRun(newAttribs, function(start, end, attribs) {
{
builder.insert(newText.substring(start, end), attribs); builder.insert(newText.substring(start, end), attribs);
}); });
// the changeset is ready! // the changeset is ready!
var theChangeset = builder.toString(); var theChangeset = builder.toString();
apiLogger.debug('The changeset: ' + theChangeset); apiLogger.debug('The changeset: ' + theChangeset);
pad.setText("\n"); pad.setText("\n");
pad.appendRevision(theChangeset); pad.appendRevision(theChangeset);
callback(null);
} }
exports.setPadHTML = setPadHTML;

58
src/node/utils/TidyHtml.js
View file

@ -6,36 +6,38 @@ var log4js = require('log4js');
var settings = require('./Settings'); var settings = require('./Settings');
var spawn = require('child_process').spawn; var spawn = require('child_process').spawn;
exports.tidy = function(srcFile, callback) { exports.tidy = function(srcFile) {
var logger = log4js.getLogger('TidyHtml'); var logger = log4js.getLogger('TidyHtml');
// Don't do anything if Tidy hasn't been enabled return new Promise((resolve, reject) => {
if (!settings.tidyHtml) {
logger.debug('tidyHtml has not been configured yet, ignoring tidy request');
return callback(null);
}
var errMessage = ''; // Don't do anything if Tidy hasn't been enabled
if (!settings.tidyHtml) {
// Spawn a new tidy instance that cleans up the file inline logger.debug('tidyHtml has not been configured yet, ignoring tidy request');
logger.debug('Tidying ' + srcFile); return resolve(null);
var tidy = spawn(settings.tidyHtml, ['-modify', srcFile]);
// Keep track of any error messages
tidy.stderr.on('data', function (data) {
errMessage += data.toString();
});
// Wait until Tidy is done
tidy.on('close', function(code) {
// Tidy returns a 0 when no errors occur and a 1 exit code when
// the file could be tidied but a few warnings were generated
if (code === 0 || code === 1) {
logger.debug('Tidied ' + srcFile + ' successfully');
return callback(null);
} else {
logger.error('Failed to tidy ' + srcFile + '\n' + errMessage);
return callback('Tidy died with exit code ' + code);
} }
var errMessage = '';
// Spawn a new tidy instance that cleans up the file inline
logger.debug('Tidying ' + srcFile);
var tidy = spawn(settings.tidyHtml, ['-modify', srcFile]);
// Keep track of any error messages
tidy.stderr.on('data', function (data) {
errMessage += data.toString();
});
tidy.on('close', function(code) {
// Tidy returns a 0 when no errors occur and a 1 exit code when
// the file could be tidied but a few warnings were generated
if (code === 0 || code === 1) {
logger.debug('Tidied ' + srcFile + ' successfully');
resolve(null);
} else {
logger.error('Failed to tidy ' + srcFile + '\n' + errMessage);
reject('Tidy died with exit code ' + code);
}
});
}); });
}; }

573
src/node/utils/padDiff.js
View file

@ -1,336 +1,267 @@
var Changeset = require("../../static/js/Changeset"); var Changeset = require("../../static/js/Changeset");
var async = require("async");
var exportHtml = require('./ExportHtml'); var exportHtml = require('./ExportHtml');
function PadDiff (pad, fromRev, toRev){ function PadDiff (pad, fromRev, toRev) {
//check parameters // check parameters
if(!pad || !pad.id || !pad.atext || !pad.pool) if (!pad || !pad.id || !pad.atext || !pad.pool) {
{
throw new Error('Invalid pad'); throw new Error('Invalid pad');
} }
var range = pad.getValidRevisionRange(fromRev, toRev); var range = pad.getValidRevisionRange(fromRev, toRev);
if(!range) { throw new Error('Invalid revision range.' + if (!range) {
throw new Error('Invalid revision range.' +
' startRev: ' + fromRev + ' startRev: ' + fromRev +
' endRev: ' + toRev); } ' endRev: ' + toRev);
}
this._pad = pad; this._pad = pad;
this._fromRev = range.startRev; this._fromRev = range.startRev;
this._toRev = range.endRev; this._toRev = range.endRev;
this._html = null; this._html = null;
this._authors = []; this._authors = [];
} }
PadDiff.prototype._isClearAuthorship = function(changeset){ PadDiff.prototype._isClearAuthorship = function(changeset) {
//unpack // unpack
var unpacked = Changeset.unpack(changeset); var unpacked = Changeset.unpack(changeset);
//check if there is nothing in the charBank // check if there is nothing in the charBank
if(unpacked.charBank !== "") if (unpacked.charBank !== "") {
return false; return false;
}
//check if oldLength == newLength
if(unpacked.oldLen !== unpacked.newLen) // check if oldLength == newLength
if (unpacked.oldLen !== unpacked.newLen) {
return false; return false;
}
//lets iterator over the operators
// lets iterator over the operators
var iterator = Changeset.opIterator(unpacked.ops); var iterator = Changeset.opIterator(unpacked.ops);
//get the first operator, this should be a clear operator // get the first operator, this should be a clear operator
var clearOperator = iterator.next(); var clearOperator = iterator.next();
//check if there is only one operator // check if there is only one operator
if(iterator.hasNext() === true) if (iterator.hasNext() === true) {
return false; return false;
}
//check if this operator doesn't change text
if(clearOperator.opcode !== "=") // check if this operator doesn't change text
if (clearOperator.opcode !== "=") {
return false; return false;
}
//check that this operator applys to the complete text
//if the text ends with a new line, its exactly one character less, else it has the same length // check that this operator applys to the complete text
if(clearOperator.chars !== unpacked.oldLen-1 && clearOperator.chars !== unpacked.oldLen) // if the text ends with a new line, its exactly one character less, else it has the same length
if (clearOperator.chars !== unpacked.oldLen-1 && clearOperator.chars !== unpacked.oldLen) {
return false; return false;
}
var attributes = []; var attributes = [];
Changeset.eachAttribNumber(changeset, function(attrNum){ Changeset.eachAttribNumber(changeset, function(attrNum) {
attributes.push(attrNum); attributes.push(attrNum);
}); });
//check that this changeset uses only one attribute // check that this changeset uses only one attribute
if(attributes.length !== 1) if (attributes.length !== 1) {
return false; return false;
}
var appliedAttribute = this._pad.pool.getAttrib(attributes[0]); var appliedAttribute = this._pad.pool.getAttrib(attributes[0]);
//check if the applied attribute is an anonymous author attribute // check if the applied attribute is an anonymous author attribute
if(appliedAttribute[0] !== "author" || appliedAttribute[1] !== "") if (appliedAttribute[0] !== "author" || appliedAttribute[1] !== "") {
return false; return false;
}
return true; return true;
}; };
PadDiff.prototype._createClearAuthorship = function(rev, callback){ PadDiff.prototype._createClearAuthorship = async function(rev) {
var self = this;
this._pad.getInternalRevisionAText(rev, function(err, atext){ let atext = await this._pad.getInternalRevisionAText(rev);
if(err){
return callback(err); // build clearAuthorship changeset
} var builder = Changeset.builder(atext.text.length);
builder.keepText(atext.text, [['author','']], this._pad.pool);
//build clearAuthorship changeset var changeset = builder.toString();
var builder = Changeset.builder(atext.text.length);
builder.keepText(atext.text, [['author','']], self._pad.pool); return changeset;
var changeset = builder.toString(); }
callback(null, changeset); PadDiff.prototype._createClearStartAtext = async function(rev) {
});
}; // get the atext of this revision
let atext = this._pad.getInternalRevisionAText(rev);
PadDiff.prototype._createClearStartAtext = function(rev, callback){
var self = this; // create the clearAuthorship changeset
let changeset = await this._createClearAuthorship(rev);
//get the atext of this revision
this._pad.getInternalRevisionAText(rev, function(err, atext){ // apply the clearAuthorship changeset
if(err){ let newAText = Changeset.applyToAText(changeset, atext, this._pad.pool);
return callback(err);
} return newAText;
}
//create the clearAuthorship changeset
self._createClearAuthorship(rev, function(err, changeset){ PadDiff.prototype._getChangesetsInBulk = async function(startRev, count) {
if(err){
return callback(err); // find out which revisions we need
} let revisions = [];
for (let i = startRev; i < (startRev + count) && i <= this._pad.head; i++) {
try {
//apply the clearAuthorship changeset
var newAText = Changeset.applyToAText(changeset, atext, self._pad.pool);
} catch(err) {
return callback(err)
}
callback(null, newAText);
});
});
};
PadDiff.prototype._getChangesetsInBulk = function(startRev, count, callback) {
var self = this;
//find out which revisions we need
var revisions = [];
for(var i=startRev;i<(startRev+count) && i<=this._pad.head;i++){
revisions.push(i); revisions.push(i);
} }
var changesets = [], authors = []; // get all needed revisions (in parallel)
let changesets = [], authors = [];
//get all needed revisions await Promise.all(revisions.map(rev => {
async.forEach(revisions, function(rev, callback){ return this._pad.getRevision(rev).then(revision => {
self._pad.getRevision(rev, function(err, revision){ let arrayNum = rev - startRev;
if(err){
return callback(err);
}
var arrayNum = rev-startRev;
changesets[arrayNum] = revision.changeset; changesets[arrayNum] = revision.changeset;
authors[arrayNum] = revision.meta.author; authors[arrayNum] = revision.meta.author;
callback();
}); });
}, function(err){ }));
callback(err, changesets, authors);
}); return { changesets, authors };
}; }
PadDiff.prototype._addAuthors = function(authors) { PadDiff.prototype._addAuthors = function(authors) {
var self = this; var self = this;
//add to array if not in the array
authors.forEach(function(author){ // add to array if not in the array
if(self._authors.indexOf(author) == -1){ authors.forEach(function(author) {
if (self._authors.indexOf(author) == -1) {
self._authors.push(author); self._authors.push(author);
} }
}); });
}; };
PadDiff.prototype._createDiffAtext = function(callback) { PadDiff.prototype._createDiffAtext = async function() {
var self = this;
var bulkSize = 100; let bulkSize = 100;
//get the cleaned startAText // get the cleaned startAText
self._createClearStartAtext(self._fromRev, function(err, atext){ let atext = await this._createClearStartAtext(this._fromRev);
if(err) { return callback(err); }
let superChangeset = null;
var superChangeset = null; let rev = this._fromRev + 1;
var rev = self._fromRev + 1; for (let rev = this._fromRev + 1; rev <= this._toRev; rev += bulkSize) {
//async while loop // get the bulk
async.whilst( let { changesets, authors } = await this._getChangesetsInBulk(rev, bulkSize);
//loop condition
function () { return rev <= self._toRev; }, let addedAuthors = [];
//loop body // run through all changesets
function (callback) { for (let i = 0; i < changesets.length && (rev + i) <= this._toRev; ++i) {
//get the bulk let changeset = changesets[i];
self._getChangesetsInBulk(rev,bulkSize,function(err, changesets, authors){
var addedAuthors = []; // skip clearAuthorship Changesets
if (this._isClearAuthorship(changeset)) {
//run trough all changesets continue;
for(var i=0;i<changesets.length && (rev+i)<=self._toRev;i++){ }
var changeset = changesets[i];
changeset = this._extendChangesetWithAuthor(changeset, authors[i], this._pad.pool);
//skip clearAuthorship Changesets
if(self._isClearAuthorship(changeset)){ // add this author to the authorarray
continue; addedAuthors.push(authors[i]);
}
// compose it with the superChangset
changeset = self._extendChangesetWithAuthor(changeset, authors[i], self._pad.pool); if (superChangeset === null) {
superChangeset = changeset;
//add this author to the authorarray } else {
addedAuthors.push(authors[i]); superChangeset = Changeset.composeWithDeletions(superChangeset, changeset, this._pad.pool);
//compose it with the superChangset
if(superChangeset === null){
superChangeset = changeset;
} else {
superChangeset = Changeset.composeWithDeletions(superChangeset, changeset, self._pad.pool);
}
}
//add the authors to the PadDiff authorArray
self._addAuthors(addedAuthors);
//lets continue with the next bulk
rev += bulkSize;
callback();
});
},
//after the loop has ended
function (err) {
//if there are only clearAuthorship changesets, we don't get a superChangeset, so we can skip this step
if(superChangeset){
var deletionChangeset = self._createDeletionChangeset(superChangeset,atext,self._pad.pool);
try {
//apply the superChangeset, which includes all addings
atext = Changeset.applyToAText(superChangeset,atext,self._pad.pool);
//apply the deletionChangeset, which adds a deletions
atext = Changeset.applyToAText(deletionChangeset,atext,self._pad.pool);
} catch(err) {
return callback(err)
}
}
callback(err, atext);
} }
);
});
};
PadDiff.prototype.getHtml = function(callback){
//cache the html
if(this._html != null){
return callback(null, this._html);
}
var self = this;
var atext, html, authorColors;
async.series([
//get the diff atext
function(callback){
self._createDiffAtext(function(err, _atext){
if(err){
return callback(err);
}
atext = _atext;
callback();
});
},
//get the authorColor table
function(callback){
self._pad.getAllAuthorColors(function(err, _authorColors){
if(err){
return callback(err);
}
authorColors = _authorColors;
callback();
});
},
//convert the atext to html
function(callback){
html = exportHtml.getHTMLFromAtext(self._pad, atext, authorColors);
self._html = html;
callback();
} }
], function(err){
callback(err, html); // add the authors to the PadDiff authorArray
}); this._addAuthors(addedAuthors);
};
PadDiff.prototype.getAuthors = function(callback){
var self = this;
//check if html was already produced, if not produce it, this generates the author array at the same time
if(self._html == null){
self.getHtml(function(err){
if(err){
return callback(err);
}
callback(null, self._authors);
});
} else {
callback(null, self._authors);
} }
};
// if there are only clearAuthorship changesets, we don't get a superChangeset, so we can skip this step
PadDiff.prototype._extendChangesetWithAuthor = function(changeset, author, apool) { if (superChangeset) {
//unpack let deletionChangeset = this._createDeletionChangeset(superChangeset, atext, this._pad.pool);
var unpacked = Changeset.unpack(changeset);
// apply the superChangeset, which includes all addings
atext = Changeset.applyToAText(superChangeset, atext, this._pad.pool);
// apply the deletionChangeset, which adds a deletions
atext = Changeset.applyToAText(deletionChangeset, atext, this._pad.pool);
}
return atext;
}
PadDiff.prototype.getHtml = async function() {
// cache the html
if (this._html != null) {
return this._html;
}
// get the diff atext
let atext = await this._createDiffAtext();
// get the authorColor table
let authorColors = await this._pad.getAllAuthorColors();
// convert the atext to html
this._html = exportHtml.getHTMLFromAtext(this._pad, atext, authorColors);
return this._html;
}
PadDiff.prototype.getAuthors = async function() {
// check if html was already produced, if not produce it, this generates the author array at the same time
if (this._html == null) {
await this.getHtml();
}
return self._authors;
}
PadDiff.prototype._extendChangesetWithAuthor = function(changeset, author, apool) {
// unpack
var unpacked = Changeset.unpack(changeset);
var iterator = Changeset.opIterator(unpacked.ops); var iterator = Changeset.opIterator(unpacked.ops);
var assem = Changeset.opAssembler(); var assem = Changeset.opAssembler();
//create deleted attribs // create deleted attribs
var authorAttrib = apool.putAttrib(["author", author || ""]); var authorAttrib = apool.putAttrib(["author", author || ""]);
var deletedAttrib = apool.putAttrib(["removed", true]); var deletedAttrib = apool.putAttrib(["removed", true]);
var attribs = "*" + Changeset.numToString(authorAttrib) + "*" + Changeset.numToString(deletedAttrib); var attribs = "*" + Changeset.numToString(authorAttrib) + "*" + Changeset.numToString(deletedAttrib);
//iteratore over the operators of the changeset // iteratore over the operators of the changeset
while(iterator.hasNext()){ while(iterator.hasNext()) {
var operator = iterator.next(); var operator = iterator.next();
//this is a delete operator, extend it with the author if (operator.opcode === "-") {
if(operator.opcode === "-"){ // this is a delete operator, extend it with the author
operator.attribs = attribs; operator.attribs = attribs;
} } else if (operator.opcode === "=" && operator.attribs) {
//this is operator changes only attributes, let's mark which author did that // this is operator changes only attributes, let's mark which author did that
else if(operator.opcode === "=" && operator.attribs){
operator.attribs+="*"+Changeset.numToString(authorAttrib); operator.attribs+="*"+Changeset.numToString(authorAttrib);
} }
//append the new operator to our assembler // append the new operator to our assembler
assem.append(operator); assem.append(operator);
} }
//return the modified changeset // return the modified changeset
return Changeset.pack(unpacked.oldLen, unpacked.newLen, assem.toString(), unpacked.charBank); return Changeset.pack(unpacked.oldLen, unpacked.newLen, assem.toString(), unpacked.charBank);
}; };
//this method is 80% like Changeset.inverse. I just changed so instead of reverting, it adds deletions and attribute changes to to the atext. // this method is 80% like Changeset.inverse. I just changed so instead of reverting, it adds deletions and attribute changes to to the atext.
PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) { PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
var lines = Changeset.splitTextLines(startAText.text); var lines = Changeset.splitTextLines(startAText.text);
var alines = Changeset.splitAttributionLines(startAText.attribs, startAText.text); var alines = Changeset.splitAttributionLines(startAText.attribs, startAText.text);
// lines and alines are what the exports is meant to apply to. // lines and alines are what the exports is meant to apply to.
// They may be arrays or objects with .get(i) and .length methods. // They may be arrays or objects with .get(i) and .length methods.
// They include final newlines on lines. // They include final newlines on lines.
function lines_get(idx) { function lines_get(idx) {
if (lines.get) { if (lines.get) {
return lines.get(idx); return lines.get(idx);
@ -338,7 +269,7 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
return lines[idx]; return lines[idx];
} }
} }
function alines_get(idx) { function alines_get(idx) {
if (alines.get) { if (alines.get) {
return alines.get(idx); return alines.get(idx);
@ -346,19 +277,19 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
return alines[idx]; return alines[idx];
} }
} }
var curLine = 0; var curLine = 0;
var curChar = 0; var curChar = 0;
var curLineOpIter = null; var curLineOpIter = null;
var curLineOpIterLine; var curLineOpIterLine;
var curLineNextOp = Changeset.newOp('+'); var curLineNextOp = Changeset.newOp('+');
var unpacked = Changeset.unpack(cs); var unpacked = Changeset.unpack(cs);
var csIter = Changeset.opIterator(unpacked.ops); var csIter = Changeset.opIterator(unpacked.ops);
var builder = Changeset.builder(unpacked.newLen); var builder = Changeset.builder(unpacked.newLen);
function consumeAttribRuns(numChars, func /*(len, attribs, endsLine)*/ ) { function consumeAttribRuns(numChars, func /*(len, attribs, endsLine)*/ ) {
if ((!curLineOpIter) || (curLineOpIterLine != curLine)) { if ((!curLineOpIter) || (curLineOpIterLine != curLine)) {
// create curLineOpIter and advance it to curChar // create curLineOpIter and advance it to curChar
curLineOpIter = Changeset.opIterator(alines_get(curLine)); curLineOpIter = Changeset.opIterator(alines_get(curLine));
@ -375,7 +306,7 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
} }
} }
} }
while (numChars > 0) { while (numChars > 0) {
if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) { if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) {
curLine++; curLine++;
@ -384,22 +315,25 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
curLineNextOp.chars = 0; curLineNextOp.chars = 0;
curLineOpIter = Changeset.opIterator(alines_get(curLine)); curLineOpIter = Changeset.opIterator(alines_get(curLine));
} }
if (!curLineNextOp.chars) { if (!curLineNextOp.chars) {
curLineOpIter.next(curLineNextOp); curLineOpIter.next(curLineNextOp);
} }
var charsToUse = Math.min(numChars, curLineNextOp.chars); var charsToUse = Math.min(numChars, curLineNextOp.chars);
func(charsToUse, curLineNextOp.attribs, charsToUse == curLineNextOp.chars && curLineNextOp.lines > 0); func(charsToUse, curLineNextOp.attribs, charsToUse == curLineNextOp.chars && curLineNextOp.lines > 0);
numChars -= charsToUse; numChars -= charsToUse;
curLineNextOp.chars -= charsToUse; curLineNextOp.chars -= charsToUse;
curChar += charsToUse; curChar += charsToUse;
} }
if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) { if ((!curLineNextOp.chars) && (!curLineOpIter.hasNext())) {
curLine++; curLine++;
curChar = 0; curChar = 0;
} }
} }
function skip(N, L) { function skip(N, L) {
if (L) { if (L) {
curLine += L; curLine += L;
@ -412,27 +346,29 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
} }
} }
} }
function nextText(numChars) { function nextText(numChars) {
var len = 0; var len = 0;
var assem = Changeset.stringAssembler(); var assem = Changeset.stringAssembler();
var firstString = lines_get(curLine).substring(curChar); var firstString = lines_get(curLine).substring(curChar);
len += firstString.length; len += firstString.length;
assem.append(firstString); assem.append(firstString);
var lineNum = curLine + 1; var lineNum = curLine + 1;
while (len < numChars) { while (len < numChars) {
var nextString = lines_get(lineNum); var nextString = lines_get(lineNum);
len += nextString.length; len += nextString.length;
assem.append(nextString); assem.append(nextString);
lineNum++; lineNum++;
} }
return assem.toString().substring(0, numChars); return assem.toString().substring(0, numChars);
} }
function cachedStrFunc(func) { function cachedStrFunc(func) {
var cache = {}; var cache = {};
return function (s) { return function (s) {
if (!cache[s]) { if (!cache[s]) {
cache[s] = func(s); cache[s] = func(s);
@ -440,57 +376,59 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
return cache[s]; return cache[s];
}; };
} }
var attribKeys = []; var attribKeys = [];
var attribValues = []; var attribValues = [];
//iterate over all operators of this changeset // iterate over all operators of this changeset
while (csIter.hasNext()) { while (csIter.hasNext()) {
var csOp = csIter.next(); var csOp = csIter.next();
if (csOp.opcode == '=') { if (csOp.opcode == '=') {
var textBank = nextText(csOp.chars); var textBank = nextText(csOp.chars);
// decide if this equal operator is an attribution change or not. We can see this by checkinf if attribs is set. // decide if this equal operator is an attribution change or not. We can see this by checkinf if attribs is set.
// If the text this operator applies to is only a star, than this is a false positive and should be ignored // If the text this operator applies to is only a star, than this is a false positive and should be ignored
if (csOp.attribs && textBank != "*") { if (csOp.attribs && textBank != "*") {
var deletedAttrib = apool.putAttrib(["removed", true]); var deletedAttrib = apool.putAttrib(["removed", true]);
var authorAttrib = apool.putAttrib(["author", ""]); var authorAttrib = apool.putAttrib(["author", ""]);
attribKeys.length = 0; attribKeys.length = 0;
attribValues.length = 0; attribValues.length = 0;
Changeset.eachAttribNumber(csOp.attribs, function (n) { Changeset.eachAttribNumber(csOp.attribs, function (n) {
attribKeys.push(apool.getAttribKey(n)); attribKeys.push(apool.getAttribKey(n));
attribValues.push(apool.getAttribValue(n)); attribValues.push(apool.getAttribValue(n));
if(apool.getAttribKey(n) === "author"){ if (apool.getAttribKey(n) === "author") {
authorAttrib = n; authorAttrib = n;
} }
}); });
var undoBackToAttribs = cachedStrFunc(function (attribs) { var undoBackToAttribs = cachedStrFunc(function (attribs) {
var backAttribs = []; var backAttribs = [];
for (var i = 0; i < attribKeys.length; i++) { for (var i = 0; i < attribKeys.length; i++) {
var appliedKey = attribKeys[i]; var appliedKey = attribKeys[i];
var appliedValue = attribValues[i]; var appliedValue = attribValues[i];
var oldValue = Changeset.attribsAttributeValue(attribs, appliedKey, apool); var oldValue = Changeset.attribsAttributeValue(attribs, appliedKey, apool);
if (appliedValue != oldValue) { if (appliedValue != oldValue) {
backAttribs.push([appliedKey, oldValue]); backAttribs.push([appliedKey, oldValue]);
} }
} }
return Changeset.makeAttribsString('=', backAttribs, apool); return Changeset.makeAttribsString('=', backAttribs, apool);
}); });
var oldAttribsAddition = "*" + Changeset.numToString(deletedAttrib) + "*" + Changeset.numToString(authorAttrib); var oldAttribsAddition = "*" + Changeset.numToString(deletedAttrib) + "*" + Changeset.numToString(authorAttrib);
var textLeftToProcess = textBank; var textLeftToProcess = textBank;
while(textLeftToProcess.length > 0){ while(textLeftToProcess.length > 0) {
//process till the next line break or process only one line break // process till the next line break or process only one line break
var lengthToProcess = textLeftToProcess.indexOf("\n"); var lengthToProcess = textLeftToProcess.indexOf("\n");
var lineBreak = false; var lineBreak = false;
switch(lengthToProcess){ switch(lengthToProcess) {
case -1: case -1:
lengthToProcess=textLeftToProcess.length; lengthToProcess=textLeftToProcess.length;
break; break;
case 0: case 0:
@ -498,27 +436,28 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
lengthToProcess=1; lengthToProcess=1;
break; break;
} }
//get the text we want to procceed in this step // get the text we want to procceed in this step
var processText = textLeftToProcess.substr(0, lengthToProcess); var processText = textLeftToProcess.substr(0, lengthToProcess);
textLeftToProcess = textLeftToProcess.substr(lengthToProcess); textLeftToProcess = textLeftToProcess.substr(lengthToProcess);
if(lineBreak){ if (lineBreak) {
builder.keep(1, 1); //just skip linebreaks, don't do a insert + keep for a linebreak builder.keep(1, 1); // just skip linebreaks, don't do a insert + keep for a linebreak
//consume the attributes of this linebreak // consume the attributes of this linebreak
consumeAttribRuns(1, function(){}); consumeAttribRuns(1, function() {});
} else { } else {
//add the old text via an insert, but add a deletion attribute + the author attribute of the author who deleted it // add the old text via an insert, but add a deletion attribute + the author attribute of the author who deleted it
var textBankIndex = 0; var textBankIndex = 0;
consumeAttribRuns(lengthToProcess, function (len, attribs, endsLine) { consumeAttribRuns(lengthToProcess, function (len, attribs, endsLine) {
//get the old attributes back // get the old attributes back
var attribs = (undoBackToAttribs(attribs) || "") + oldAttribsAddition; var attribs = (undoBackToAttribs(attribs) || "") + oldAttribsAddition;
builder.insert(processText.substr(textBankIndex, len), attribs); builder.insert(processText.substr(textBankIndex, len), attribs);
textBankIndex += len; textBankIndex += len;
}); });
builder.keep(lengthToProcess, 0); builder.keep(lengthToProcess, 0);
} }
} }
@ -531,16 +470,16 @@ PadDiff.prototype._createDeletionChangeset = function(cs, startAText, apool) {
} else if (csOp.opcode == '-') { } else if (csOp.opcode == '-') {
var textBank = nextText(csOp.chars); var textBank = nextText(csOp.chars);
var textBankIndex = 0; var textBankIndex = 0;
consumeAttribRuns(csOp.chars, function (len, attribs, endsLine) { consumeAttribRuns(csOp.chars, function (len, attribs, endsLine) {
builder.insert(textBank.substr(textBankIndex, len), attribs + csOp.attribs); builder.insert(textBank.substr(textBankIndex, len), attribs + csOp.attribs);
textBankIndex += len; textBankIndex += len;
}); });
} }
} }
return Changeset.checkRep(builder.toString()); return Changeset.checkRep(builder.toString());
}; };
//export the constructor // export the constructor
module.exports = PadDiff; module.exports = PadDiff;

2
src/package.json
View file

@ -48,6 +48,7 @@
"languages4translatewiki": "0.1.3", "languages4translatewiki": "0.1.3",
"log4js": "0.6.35", "log4js": "0.6.35",
"measured-core": "1.11.2", "measured-core": "1.11.2",
"nodeify": "^1.0.1",
"npm": "6.4.1", "npm": "6.4.1",
"object.values": "^1.0.4", "object.values": "^1.0.4",
"request": "2.88.0", "request": "2.88.0",
@ -86,4 +87,3 @@
"version": "1.7.5", "version": "1.7.5",
"license": "Apache-2.0" "license": "Apache-2.0"
} }

30
src/static/js/pluginfw/hooks.js
View file

@ -78,7 +78,7 @@ exports.callAll = function (hook_name, args) {
} }
} }
exports.aCallAll = function (hook_name, args, cb) { function aCallAll(hook_name, args, cb) {
if (!args) args = {}; if (!args) args = {};
if (!cb) cb = function () {}; if (!cb) cb = function () {};
if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []); if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []);
@ -93,6 +93,19 @@ exports.aCallAll = function (hook_name, args, cb) {
); );
} }
/* return a Promise if cb is not supplied */
exports.aCallAll = function (hook_name, args, cb) {
if (cb === undefined) {
return new Promise(function(resolve, reject) {
aCallAll(hook_name, args, function(err, res) {
return err ? reject(err) : resolve(res);
});
});
} else {
return aCallAll(hook_name, args, cb);
}
}
exports.callFirst = function (hook_name, args) { exports.callFirst = function (hook_name, args) {
if (!args) args = {}; if (!args) args = {};
if (exports.plugins.hooks[hook_name] === undefined) return []; if (exports.plugins.hooks[hook_name] === undefined) return [];
@ -101,7 +114,7 @@ exports.callFirst = function (hook_name, args) {
}); });
} }
exports.aCallFirst = function (hook_name, args, cb) { function aCallFirst(hook_name, args, cb) {
if (!args) args = {}; if (!args) args = {};
if (!cb) cb = function () {}; if (!cb) cb = function () {};
if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []); if (exports.plugins.hooks[hook_name] === undefined) return cb(null, []);
@ -114,6 +127,19 @@ exports.aCallFirst = function (hook_name, args, cb) {
); );
} }
/* return a Promise if cb is not supplied */
exports.aCallFirst = function (hook_name, args, cb) {
if (cb === undefined) {
return new Promise(function(resolve, reject) {
aCallFirst(hook_name, args, function(err, res) {
return err ? reject(err) : resolve(res);
});
});
} else {
return aCallFirst(hook_name, args, cb);
}
}
exports.callAllStr = function(hook_name, args, sep, pre, post) { exports.callAllStr = function(hook_name, args, sep, pre, post) {
if (sep == undefined) sep = ''; if (sep == undefined) sep = '';
if (pre == undefined) pre = ''; if (pre == undefined) pre = '';

119
src/static/js/pluginfw/installer.js
View file

@ -4,12 +4,14 @@ var npm = require("npm");
var request = require("request"); var request = require("request");
var npmIsLoaded = false; var npmIsLoaded = false;
var withNpm = function (npmfn) { var withNpm = function(npmfn) {
if(npmIsLoaded) return npmfn(); if (npmIsLoaded) return npmfn();
npm.load({}, function (er) {
npm.load({}, function(er) {
if (er) return npmfn(er); if (er) return npmfn(er);
npmIsLoaded = true; npmIsLoaded = true;
npm.on("log", function (message) { npm.on("log", function(message) {
console.log('npm: ',message) console.log('npm: ',message)
}); });
npmfn(); npmfn();
@ -17,42 +19,57 @@ var withNpm = function (npmfn) {
} }
var tasks = 0 var tasks = 0
function wrapTaskCb(cb) { function wrapTaskCb(cb) {
tasks++ tasks++;
return function() { return function() {
cb && cb.apply(this, arguments); cb && cb.apply(this, arguments);
tasks--; tasks--;
if(tasks == 0) onAllTasksFinished(); if (tasks == 0) onAllTasksFinished();
} }
} }
function onAllTasksFinished() { function onAllTasksFinished() {
hooks.aCallAll("restartServer", {}, function () {}); hooks.aCallAll("restartServer", {}, function() {});
} }
/*
* We cannot use arrow functions in this file, because code in /src/static
* can end up being loaded in browsers, and we still support IE11.
*/
exports.uninstall = function(plugin_name, cb) { exports.uninstall = function(plugin_name, cb) {
cb = wrapTaskCb(cb); cb = wrapTaskCb(cb);
withNpm(function (er) {
withNpm(function(er) {
if (er) return cb && cb(er); if (er) return cb && cb(er);
npm.commands.uninstall([plugin_name], function (er) {
npm.commands.uninstall([plugin_name], function(er) {
if (er) return cb && cb(er); if (er) return cb && cb(er);
hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name}, function (er, data) { hooks.aCallAll("pluginUninstall", {plugin_name: plugin_name})
if (er) return cb(er); .then(plugins.update)
plugins.update(cb); .then(function() { cb(null) })
}); .catch(function(er) { cb(er) });
}); });
}); });
}; };
/*
* We cannot use arrow functions in this file, because code in /src/static
* can end up being loaded in browsers, and we still support IE11.
*/
exports.install = function(plugin_name, cb) { exports.install = function(plugin_name, cb) {
cb = wrapTaskCb(cb) cb = wrapTaskCb(cb);
withNpm(function (er) {
withNpm(function(er) {
if (er) return cb && cb(er); if (er) return cb && cb(er);
npm.commands.install([plugin_name], function (er) {
npm.commands.install([plugin_name], function(er) {
if (er) return cb && cb(er); if (er) return cb && cb(er);
hooks.aCallAll("pluginInstall", {plugin_name: plugin_name}, function (er, data) { hooks.aCallAll("pluginInstall", {plugin_name: plugin_name})
if (er) return cb(er); .then(plugins.update)
plugins.update(cb); .then(function() { cb(null) })
}); .catch(function(er) { cb(er) });
}); });
}); });
}; };
@ -60,44 +77,58 @@ exports.install = function(plugin_name, cb) {
exports.availablePlugins = null; exports.availablePlugins = null;
var cacheTimestamp = 0; var cacheTimestamp = 0;
exports.getAvailablePlugins = function(maxCacheAge, cb) { exports.getAvailablePlugins = function(maxCacheAge) {
request("https://static.etherpad.org/plugins.json", function(er, response, plugins){ var nowTimestamp = Math.round(Date.now() / 1000);
if (er) return cb && cb(er);
if(exports.availablePlugins && maxCacheAge && Math.round(+new Date/1000)-cacheTimestamp <= maxCacheAge) { return new Promise(function(resolve, reject) {
return cb && cb(null, exports.availablePlugins) // check cache age before making any request
if (exports.availablePlugins && maxCacheAge && (nowTimestamp - cacheTimestamp) <= maxCacheAge) {
return resolve(exports.availablePlugins);
} }
try {
plugins = JSON.parse(plugins); request("https://static.etherpad.org/plugins.json", function(er, response, plugins) {
} catch (err) { if (er) return reject(er);
console.error('error parsing plugins.json:', err);
plugins = []; try {
} plugins = JSON.parse(plugins);
exports.availablePlugins = plugins; } catch (err) {
cacheTimestamp = Math.round(+new Date/1000); console.error('error parsing plugins.json:', err);
cb && cb(null, plugins) plugins = [];
}
exports.availablePlugins = plugins;
cacheTimestamp = nowTimestamp;
resolve(plugins);
});
}); });
}; };
exports.search = function(searchTerm, maxCacheAge, cb) { exports.search = function(searchTerm, maxCacheAge) {
exports.getAvailablePlugins(maxCacheAge, function(er, results) { return exports.getAvailablePlugins(maxCacheAge).then(function(results) {
if(er) return cb && cb(er);
var res = {}; var res = {};
if (searchTerm)
if (searchTerm) {
searchTerm = searchTerm.toLowerCase(); searchTerm = searchTerm.toLowerCase();
for (var pluginName in results) { // for every available plugin }
for (var pluginName in results) {
// for every available plugin
if (pluginName.indexOf(plugins.prefix) != 0) continue; // TODO: Also search in keywords here! if (pluginName.indexOf(plugins.prefix) != 0) continue; // TODO: Also search in keywords here!
if(searchTerm && !~results[pluginName].name.toLowerCase().indexOf(searchTerm) if (searchTerm && !~results[pluginName].name.toLowerCase().indexOf(searchTerm)
&& (typeof results[pluginName].description != "undefined" && !~results[pluginName].description.toLowerCase().indexOf(searchTerm) ) && (typeof results[pluginName].description != "undefined" && !~results[pluginName].description.toLowerCase().indexOf(searchTerm) )
){ ) {
if(typeof results[pluginName].description === "undefined"){ if (typeof results[pluginName].description === "undefined") {
console.debug('plugin without Description: %s', results[pluginName].name); console.debug('plugin without Description: %s', results[pluginName].name);
} }
continue; continue;
} }
res[pluginName] = results[pluginName]; res[pluginName] = results[pluginName];
} }
cb && cb(null, res)
}) return res;
});
}; };

162
src/static/js/pluginfw/plugins.js
View file

@ -1,7 +1,6 @@
var npm = require("npm/lib/npm.js"); var npm = require("npm/lib/npm.js");
var readInstalled = require("./read-installed.js"); var readInstalled = require("./read-installed.js");
var path = require("path"); var path = require("path");
var async = require("async");
var fs = require("fs"); var fs = require("fs");
var tsort = require("./tsort"); var tsort = require("./tsort");
var util = require("util"); var util = require("util");
@ -15,6 +14,7 @@ exports.plugins = {};
exports.parts = []; exports.parts = [];
exports.hooks = {}; exports.hooks = {};
// @TODO RPB this appears to be unused
exports.ensure = function (cb) { exports.ensure = function (cb) {
if (!exports.loaded) if (!exports.loaded)
exports.update(cb); exports.update(cb);
@ -53,106 +53,94 @@ exports.formatHooks = function (hook_set_name) {
return "<dl>" + res.join("\n") + "</dl>"; return "<dl>" + res.join("\n") + "</dl>";
}; };
exports.callInit = function (cb) { exports.callInit = function () {
const fsp_stat = util.promisify(fs.stat);
const fsp_writeFile = util.promisify(fs.writeFile);
var hooks = require("./hooks"); var hooks = require("./hooks");
async.map(
Object.keys(exports.plugins), let p = Object.keys(exports.plugins).map(function (plugin_name) {
function (plugin_name, cb) { let plugin = exports.plugins[plugin_name];
var plugin = exports.plugins[plugin_name]; let ep_init = path.normalize(path.join(plugin.package.path, ".ep_initialized"));
fs.stat(path.normalize(path.join(plugin.package.path, ".ep_initialized")), function (err, stats) { return fsp_stat(ep_init).catch(async function() {
if (err) { await fsp_writeFile(ep_init, "done");
async.waterfall([ await hooks.aCallAll("init_" + plugin_name, {});
function (cb) { fs.writeFile(path.normalize(path.join(plugin.package.path, ".ep_initialized")), 'done', cb); }, });
function (cb) { hooks.aCallAll("init_" + plugin_name, {}, cb); }, });
cb,
]); return Promise.all(p);
} else {
cb();
}
});
},
function () { cb(); }
);
} }
exports.pathNormalization = function (part, hook_fn_name) { exports.pathNormalization = function (part, hook_fn_name) {
return path.normalize(path.join(path.dirname(exports.plugins[part.plugin].package.path), hook_fn_name)); return path.normalize(path.join(path.dirname(exports.plugins[part.plugin].package.path), hook_fn_name));
} }
exports.update = function (cb) { exports.update = async function () {
exports.getPackages(function (er, packages) { let packages = await exports.getPackages();
var parts = []; var parts = [];
var plugins = {}; var plugins = {};
// Load plugin metadata ep.json
async.forEach(
Object.keys(packages),
function (plugin_name, cb) {
loadPlugin(packages, plugin_name, plugins, parts, cb);
},
function (err) {
if (err) cb(err);
exports.plugins = plugins;
exports.parts = sortParts(parts);
exports.hooks = pluginUtils.extractHooks(exports.parts, "hooks", exports.pathNormalization);
exports.loaded = true;
exports.callInit(cb);
}
);
});
};
exports.getPackages = function (cb) { // Load plugin metadata ep.json
let p = Object.keys(packages).map(function (plugin_name) {
return loadPlugin(packages, plugin_name, plugins, parts);
});
return Promise.all(p).then(function() {
exports.plugins = plugins;
exports.parts = sortParts(parts);
exports.hooks = pluginUtils.extractHooks(exports.parts, "hooks", exports.pathNormalization);
exports.loaded = true;
}).then(exports.callInit);
}
exports.getPackages = async function () {
// Load list of installed NPM packages, flatten it to a list, and filter out only packages with names that // Load list of installed NPM packages, flatten it to a list, and filter out only packages with names that
var dir = path.resolve(npm.dir, '..'); var dir = path.resolve(npm.dir, '..');
readInstalled(dir, function (er, data) { let data = await util.promisify(readInstalled)(dir);
if (er) cb(er, null);
var packages = {}; var packages = {};
function flatten(deps) { function flatten(deps) {
_.chain(deps).keys().each(function (name) { _.chain(deps).keys().each(function (name) {
if (name.indexOf(exports.prefix) === 0) { if (name.indexOf(exports.prefix) === 0) {
packages[name] = _.clone(deps[name]); packages[name] = _.clone(deps[name]);
// Delete anything that creates loops so that the plugin // Delete anything that creates loops so that the plugin
// list can be sent as JSON to the web client // list can be sent as JSON to the web client
delete packages[name].dependencies; delete packages[name].dependencies;
delete packages[name].parent; delete packages[name].parent;
} }
// I don't think we need recursion // I don't think we need recursion
//if (deps[name].dependencies !== undefined) flatten(deps[name].dependencies); //if (deps[name].dependencies !== undefined) flatten(deps[name].dependencies);
}); });
} }
var tmp = {}; var tmp = {};
tmp[data.name] = data; tmp[data.name] = data;
flatten(tmp[data.name].dependencies); flatten(tmp[data.name].dependencies);
cb(null, packages); return packages;
});
}; };
function loadPlugin(packages, plugin_name, plugins, parts, cb) { async function loadPlugin(packages, plugin_name, plugins, parts) {
let fsp_readFile = util.promisify(fs.readFile);
var plugin_path = path.resolve(packages[plugin_name].path, "ep.json"); var plugin_path = path.resolve(packages[plugin_name].path, "ep.json");
fs.readFile( try {
plugin_path, let data = await fsp_readFile(plugin_path);
function (er, data) { try {
if (er) { var plugin = JSON.parse(data);
console.error("Unable to load plugin definition file " + plugin_path); plugin['package'] = packages[plugin_name];
return cb(); plugins[plugin_name] = plugin;
} _.each(plugin.parts, function (part) {
try { part.plugin = plugin_name;
var plugin = JSON.parse(data); part.full_name = plugin_name + "/" + part.name;
plugin['package'] = packages[plugin_name]; parts[part.full_name] = part;
plugins[plugin_name] = plugin; });
_.each(plugin.parts, function (part) { } catch (ex) {
part.plugin = plugin_name; console.error("Unable to parse plugin definition file " + plugin_path + ": " + ex.toString());
part.full_name = plugin_name + "/" + part.name;
parts[part.full_name] = part;
});
} catch (ex) {
console.error("Unable to parse plugin definition file " + plugin_path + ": " + ex.toString());
}
cb();
} }
); } catch (er) {
console.error("Unable to load plugin definition file " + plugin_path);
}
} }
function partsToParentChildList(parts) { function partsToParentChildList(parts) {

10
tests/backend/specs/api/tidy.js
View file

@ -1,10 +1,12 @@
var assert = require('assert') var assert = require('assert')
os = require('os'),
fs = require('fs'), fs = require('fs'),
path = require('path'), path = require('path'),
TidyHtml = null, TidyHtml = null,
Settings = null; Settings = null;
var npm = require("../../../../src/node_modules/npm/lib/npm.js"); var npm = require("../../../../src/node_modules/npm/lib/npm.js");
var nodeify = require('../../../../src/node_modules/nodeify');
describe('tidyHtml', function() { describe('tidyHtml', function() {
before(function(done) { before(function(done) {
@ -16,6 +18,10 @@ describe('tidyHtml', function() {
}); });
}); });
function tidy(file, callback) {
return nodeify(TidyHtml.tidy(file), callback);
}
it('Tidies HTML', function(done) { it('Tidies HTML', function(done) {
// If the user hasn't configured Tidy, we skip this tests as it's required for this test // If the user hasn't configured Tidy, we skip this tests as it's required for this test
if (!Settings.tidyHtml) { if (!Settings.tidyHtml) {
@ -27,7 +33,7 @@ describe('tidyHtml', function() {
var tmpFile = path.join(tmpDir, 'tmp_' + (Math.floor(Math.random() * 1000000)) + '.html') var tmpFile = path.join(tmpDir, 'tmp_' + (Math.floor(Math.random() * 1000000)) + '.html')
fs.writeFileSync(tmpFile, '<html><body><p>a paragraph</p><li>List without outer UL</li>trailing closing p</p></body></html>'); fs.writeFileSync(tmpFile, '<html><body><p>a paragraph</p><li>List without outer UL</li>trailing closing p</p></body></html>');
TidyHtml.tidy(tmpFile, function(err){ tidy(tmpFile, function(err){
assert.ok(!err); assert.ok(!err);
// Read the file again // Read the file again
@ -56,7 +62,7 @@ describe('tidyHtml', function() {
this.skip(); this.skip();
} }
TidyHtml.tidy('/some/none/existing/file.html', function(err) { tidy('/some/none/existing/file.html', function(err) {
assert.ok(err); assert.ok(err);
return done(); return done();
}); });