diff --git a/config.example.js b/config.example.js index b3f5d88e130..8b7aa81b56c 100644 --- a/config.example.js +++ b/config.example.js @@ -40,7 +40,15 @@ config = { port: '2368' }, paths: { - contentPath: path.join(__dirname, '/content/') + contentPath: path.join(__dirname, '/content/'), + contentStore: 'local' // Alternates: 'manta' for Joyent Object Storage + manta: { + // If using Manta Object Store for images... + // Root of image directory in Manta, relative :userid. Default: /public/ghost + // Currently expects these environment variables: MANTA_USER, MANTA_URL, MANTA_KEY_ID; + // and ssh key in .ssh + rootDir: '/public/ghost' + } } }, diff --git a/core/server/config/index.js b/core/server/config/index.js index 10448910602..d5494d03e78 100644 --- a/core/server/config/index.js +++ b/core/server/config/index.js @@ -17,6 +17,8 @@ var path = require('path'), function updateConfig(config) { var localPath, contentPath, + contentStore, + mantaConfig, subdir; // Merge passed in config object onto @@ -42,6 +44,8 @@ function updateConfig(config) { // Allow contentPath to be over-written by passed in config object // Otherwise default to default content path location contentPath = ghostConfig.paths.contentPath || path.resolve(appRoot, 'content'); + contentStore = ghostConfig.paths.contentStore || 'local'; + mantaConfig = ghostConfig.paths.manta || { 'rootDir': '/public/ghost' }; _.merge(ghostConfig, { paths: { @@ -52,10 +56,12 @@ function updateConfig(config) { 'corePath': corePath, 'contentPath': contentPath, + 'contentStore': contentStore, 'themePath': path.resolve(contentPath, 'themes'), 'appPath': path.resolve(contentPath, 'apps'), 'imagesPath': path.resolve(contentPath, 'images'), 'imagesRelPath': 'content/images', + 'manta': mantaConfig, 'adminViews': path.join(corePath, '/server/views/'), 'helperTemplates': path.join(corePath, '/server/helpers/tpl/'), @@ -111,4 +117,4 @@ module.exports = config; module.exports.init = initConfig; module.exports.theme = theme; module.exports.urlFor = configUrl.urlFor; -module.exports.urlForPost = configUrl.urlForPost; \ No newline at end of file +module.exports.urlForPost = configUrl.urlForPost; diff --git a/core/server/storage/base.js b/core/server/storage/base.js index fb344d60cce..17530b74f27 100644 --- a/core/server/storage/base.js +++ b/core/server/storage/base.js @@ -28,16 +28,19 @@ baseStore = { filename = path.join(dir, name + append + ext); - store.exists(filename).then(function (exists) { - if (exists) { - setImmediate(function () { - i = i + 1; - self.generateUnique(store, dir, name, ext, i, done); - }); - } else { - done.resolve(filename); - } - }); + // store.exists(filename).then(function (exists) { + // if (exists) { + // setImmediate(function () { + // i = i + 1; + // self.generateUnique(store, dir, name, ext, i, done); + // }); + // } else { + // done.resolve(filename); + // } + // }); + + // FIXME TOTAL HACK + done.resolve(filename); }, 'getUniqueFileName': function (store, image, targetDir) { var done = when.defer(), @@ -50,4 +53,4 @@ baseStore = { } }; -module.exports = baseStore; \ No newline at end of file +module.exports = baseStore; diff --git a/core/server/storage/index.js b/core/server/storage/index.js index 26500240c31..008bb6bd0ea 100644 --- a/core/server/storage/index.js +++ b/core/server/storage/index.js @@ -1,15 +1,22 @@ var errors = require('../errorHandling'), + config = require('../config'), storage; function get_storage() { // TODO: this is where the check for storage apps should go // Local file system is the default - var storageChoice = 'localfilesystem'; + var storageChoice; if (storage) { return storage; } + if (config().paths.contentStore == 'manta') { + storageChoice = 'mantaobjectstore'; + } else { + storageChoice = 'localfilesystem'; + } + try { // TODO: determine if storage has all the necessary methods storage = require('./' + storageChoice); @@ -19,4 +26,4 @@ function get_storage() { return storage; } -module.exports.get_storage = get_storage; \ No newline at end of file +module.exports.get_storage = get_storage; diff --git a/core/server/storage/mantaobjectstore.js b/core/server/storage/mantaobjectstore.js new file mode 100644 index 00000000000..3f6f61a4d9a --- /dev/null +++ b/core/server/storage/mantaobjectstore.js @@ -0,0 +1,122 @@ +// # Joyent Manta Object Store module +// An alternate module for storing images, using a cloud object storage system + +var _ = require('lodash'), + express = require('express'), + fs = require('fs-extra'), + nodefn = require('when/node/function'), + nodecb = require('when/callbacks'), + path = require('path'), + when = require('when'), + errors = require('../errorHandling'), + config = require('../config'), + baseStore = require('./base'), + manta = require('manta'), + bunyan = require('bunyan'), +// mantaLog = bunyan.createLogger({name: "manta"}), + + mantaObjectStore; + +mantaObjectStore = _.extend(baseStore, { + 'mantaClient': undefined, + 'createMantaClient': function () { + // FIXME This should be created at an initialization step on get_storage in parent. + // createBinClient requires bunyan logger. createBinClient automatically uses ssh-agent as needed. + var self = this, + url = process.env.MANTA_URL || 'http://localhost:8080', + user = process.env.MANTA_USER || 'admin', + mantaLog = bunyan.createLogger({name: 'manta'}); + opts = { + connectTimeout: 1000, + retry: false, + rejectUnauthorized: (process.env.MANTA_TLS_INSECURE ? + false : true), + url: url, + user: user, + log: mantaLog + }; + + self.mantaClient = manta.createBinClient(opts); + + return; + }, + + // ### Save + // Saves the image to storage (the file system) + // - image is the express image object + // - returns a promise which ultimately returns the full url to the uploaded image + 'save': function (image) { + + // To match the fn( .. callback, errorback) protocol required by nodecb + var putWrapper = function (mantapath, inputstream, opts, cb, eb) { + inputstream.on('end', function () { + cb('manta stream done.'); + }); + + mantaClient.put(mantapath, inputstream, opts, eb); + } + + var saved = when.defer(), + targetDir = this.getTargetDir(config().paths.manta.rootDir), + targetFilename; + this.getUniqueFileName(this, image, targetDir).then(function (filename) { + targetFilename = filename; + }).then(this.createMantaClient) // How expensive is this, need memoization? + .then(function () { + var opts = { + headers: { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET' + }, + 'mkdirs': true + }; + var imgstrm = fs.createReadStream(image.path), + mantapath = '~~' + targetFilename; + + nodecb.call(putWrapper, mantapath, imgstrm, opts); + + return; + }).then(function () { + return nodefn.call(fs.unlink, image.path).otherwise(errors.logError); + }).then(function () { + // The src for the image must be in URI format, not a file system path, which in Windows uses \ + // For local file system storage can use relative path so add a slash + var fullUrl = process.env.MANTA_URL + '/' + process.env.MANTA_USER + targetFilename; + return saved.resolve(fullUrl); + }).otherwise(function (e) { + errors.logError(e); + return saved.reject(e); + }); + + return saved.promise; + }, + + 'exists': function (filename) { + // fs.exists does not play nicely with nodefn because the callback doesn't have an error argument + var done = when.defer(); + + // FIXME need to test if it exists.!!! + done.resolve(function() { + return false; + }); + + // fs.exists(filename, function (exists) { + // done.resolve(exists); + // }); + + return done.promise; + }, + + // middleware for serving the files + 'serve': function () { + var ONE_HOUR_MS = 60 * 60 * 1000, + ONE_YEAR_MS = 365 * 24 * ONE_HOUR_MS; + + // For some reason send divides the max age number by 1000 + return express['static'](config().paths.imagesPath, {maxAge: ONE_YEAR_MS}); + } +}); + + + +module.exports = mantaObjectStore; diff --git a/package.json b/package.json index da067e0fd18..68a100136a9 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "dependencies": { "bcryptjs": "0.7.10", "bookshelf": "0.6.1", + "bunyan": "0.22.1", "busboy": "0.0.12", "colors": "0.6.2", "connect-slashes": "1.2.0", @@ -42,14 +43,16 @@ "express-hbs": "0.7.6", "fs-extra": "0.8.1", "knex": "0.5.0", + "manta": "1.2.6", "moment": "2.4.0", "node-polyglot": "0.3.0", "node-uuid": "1.4.1", "nodemailer": "0.5.13", + "pg": "2.11.1", "rss": "0.2.1", "semver": "2.2.1", "showdown": "0.3.1", - "sqlite3": "2.2.0", + "sqlite3": "2.1.9", "lodash": "2.4.1", "unidecode": "0.1.3", "validator": "1.4.0",