diff --git a/README.md b/README.md index 4b02de0..cd7d7c0 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ The exact JSON document provided to `PUT /content` will be returned. An HTTP status of 404 will be returned if the content ID isn't recognized. -### `POST /asset` +### `POST /asset[?named=true]` Fingerprint and publish one or more static assets to a CDN-enabled Cloud Files container. Return the full URLs to the published assets. @@ -93,6 +93,8 @@ Fingerprint and publish one or more static assets to a CDN-enabled Cloud Files c The request payload must be a `multipart/form-data` file upload containing the assets to upload. The content type of each file must be set appropriately. +If the query parameter `named=true` is provided, each asset's CDN URI will also be persisted in the *layout asset map* with a key derived from its form name and inserted in the `assets` object of every outgoing metadata envelope. + *Response* ```json diff --git a/app.js b/app.js index a56c501..300c644 100644 --- a/app.js +++ b/app.js @@ -22,7 +22,7 @@ var server.name = config.info.name; -connection.setup(config, function (err) { +connection.setup(function (err) { if (err) { throw err; } diff --git a/docker-compose.yml b/docker-compose.yml index 2291587..7095689 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,17 @@ --- +mongo: + image: mongo:2.6 content: build: . ports: - "9000:8080" + links: + - "mongo:mongo" environment: RACKSPACE_USERNAME: RACKSPACE_APIKEY: RACKSPACE_REGION: DFW CONTENT_CONTAINER: deconst-dev-content ASSET_CONTAINER: deconst-dev-assets + MONGODB_URL: mongodb://mongo:27017/content CONTENT_LOG_LEVEL: debug diff --git a/package.json b/package.json index 853b114..017f8d5 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "license": "MIT", "dependencies": { "async": "0.9.0", + "mongodb": "2.0.27", "pkgcloud": "1.1.0", "restify": "3.0.0", "winston": "0.9.0" diff --git a/script/mongo-cli b/script/mongo-cli new file mode 100755 index 0000000..898a842 --- /dev/null +++ b/script/mongo-cli @@ -0,0 +1,4 @@ +#!/bin/bash + +exec docker-compose run --rm mongo \ + sh -c 'exec mongo "${MONGO_PORT_27017_TCP_ADDR}:${MONGO_PORT_27017_TCP_PORT}/content"' diff --git a/src/assets.js b/src/assets.js index de371dd..9461af1 100644 --- a/src/assets.js +++ b/src/assets.js @@ -16,7 +16,7 @@ var log = logging.getLogger(config.content_log_level()); * @description Calculate a checksum of an uploaded file's contents to generate * the fingerprinted asset name. */ -function fingerprint(asset, callback) { +function fingerprint_asset(asset, callback) { var sha256sum = crypto.createHash('sha256'), asset_file = fs.createReadStream(asset.path), @@ -39,6 +39,7 @@ function fingerprint(asset, callback) { log.debug("Fingerprinted asset [" + asset.name + "] as [" + fingerprinted + "]."); callback(null, { + key: asset.key, original: asset.name, chunks: chunks, filename: fingerprinted, @@ -50,7 +51,7 @@ function fingerprint(asset, callback) { /** * @description Upload an asset's contents to the asset container. */ -function publish(asset, callback) { +function publish_asset(asset, callback) { var up = connection.client.upload({ container: config.asset_container(), remote: asset.filename, @@ -63,6 +64,8 @@ function publish(asset, callback) { up.on('finish', function () { log.debug("Successfully uploaded asset [" + asset.filename + "]."); + var base_uri = connection.asset_container.cdnSslUri; + asset.public_url = base_uri + '/' + encodeURIComponent(asset.filename); callback(null, asset); }); @@ -74,15 +77,39 @@ function publish(asset, callback) { } /** - * @description Process a single asset. + * @description Give this asset a name. The final name and CDL URI of this + * asset will be included in all outgoing metadata envelopes, for use by + * layouts. */ -function handle_asset(asset, callback) { - log.debug("Processing uploaded asset [" + asset.name + "]."); +function name_asset(asset, callback) { + log.debug("Naming asset [" + asset.original + "] as [" + asset.key + "]."); + + connection.db.collection("layout_assets").updateOne( + { key: asset.key }, + { $set: { key: asset.key, public_url: asset.public_url } }, + { upsert: true }, + function (err) { callback(err, asset); } + ); +} + +/** + * @description Create and return a function that processes a single asset. + */ +function make_asset_handler(should_name) { + return function(asset, callback) { + log.debug("Processing uploaded asset [" + asset.name + "]."); + + var steps = [ + async.apply(fingerprint_asset, asset), + publish_asset + ]; - async.waterfall([ - async.apply(fingerprint, asset), - publish - ], callback); + if (should_name) { + steps.push(name_asset); + } + + async.waterfall(steps, callback); + }; } /** @@ -92,16 +119,12 @@ function handle_asset(asset, callback) { */ exports.accept = function (req, res, next) { var asset_data = Object.getOwnPropertyNames(req.files).map(function (key) { - return req.files[key]; + var asset = req.files[key]; + asset.key = key; + return asset; }); - var base_uri = connection.asset_container.cdnSslUri; - - if (! base_uri) { - log.error("Asset container does not have a CDN URI. Is it CDN-enabled?"); - } - - async.map(asset_data, handle_asset, function (err, results) { + async.map(asset_data, make_asset_handler(req.query.named), function (err, results) { if (err) { log.error("Unable to process an asset.", err); @@ -114,9 +137,7 @@ exports.accept = function (req, res, next) { var summary = {}; results.forEach(function (result) { - var public_url = base_uri + '/' + encodeURIComponent(result.filename); - - summary[result.original] = public_url; + summary[result.original] = result.public_url; }); log.debug("All assets have been processed succesfully.", summary); diff --git a/src/config.js b/src/config.js index 5c022f0..aa84f64 100644 --- a/src/config.js +++ b/src/config.js @@ -11,6 +11,7 @@ var configuration = { rackspace_region: null, content_container: null, asset_container: null, + mongodb_url: null, content_log_level: "info" }; diff --git a/src/connection.js b/src/connection.js index e64c239..8c465d5 100644 --- a/src/connection.js +++ b/src/connection.js @@ -3,6 +3,7 @@ var async = require('async'), pkgcloud = require('pkgcloud'), + mongo = require('mongodb'), config = require('./config'), logging = require('./logging'); @@ -55,7 +56,25 @@ function refresh(client, container_name, logical_name, callback) { }); } -exports.setup = function (config, callback) { +/** + * @description Authenticate to MongoDB and export the active MongoDB connection as "db". + */ +function mongo_auth(callback) { + mongo.MongoClient.connect(config.mongodb_url(), function (err, db) { + if (err) { + callback(err); + return; + } + + log.debug("Connected to MongoDB database at [" + config.mongodb_url() + "]."); + + exports.db = db; + + callback(null); + }); +} + +exports.setup = function (callback) { var client = pkgcloud.providers.rackspace.storage.createClient({ username: config.rackspace_username(), apiKey: config.rackspace_apikey(), @@ -66,5 +85,6 @@ exports.setup = function (config, callback) { async.parallel([ make_container_creator(client, config.content_container(), "content_container", false), make_container_creator(client, config.asset_container(), "asset_container", true), + mongo_auth ], callback); }; diff --git a/src/content.js b/src/content.js index d125136..7db5fec 100644 --- a/src/content.js +++ b/src/content.js @@ -1,29 +1,91 @@ // Store, retrieve, and delete metadata envelopes. var + async = require('async'), config = require('./config'), connection = require('./connection'), logging = require('./logging'); var log = logging.getLogger(config.content_log_level()); +/** + * @description Download the raw metadata envelope from Cloud Files. + */ +function download_content(content_id, callback) { + var source = connection.client.download({ + container: config.content_container(), + remote: encodeURIComponent(content_id) + }); + var chunks = []; + + source.on('error', function (err) { + callback(err); + }); + + source.on('data', function (chunk) { + chunks.push(chunk); + }); + + source.on('end', function () { + var + complete = Buffer.concat(chunks), + envelope = JSON.parse(complete); + + callback(null, envelope); + }); +} + +/** + * @description Inject asset variables included from the /assets endpoint into + * an outgoing metadata envelope. + */ +function inject_asset_vars(envelope, callback) { + log.debug("Collecting asset variables to inject into the envelope."); + + connection.db.collection("layout_assets").find().toArray(function (err, asset_vars) { + if (err) { + callback(err); + return; + } + + log.debug("Injecting " + asset_vars.length + " variables into the envelope."); + + var assets = {}; + + asset_vars.forEach(function (asset_var) { + assets[asset_var.key] = asset_var.public_url; + }); + + envelope.assets = assets; + + callback(null, envelope); + }); +} + /** * @description Retrieve content from the store by content ID. */ exports.retrieve = function (req, res, next) { log.debug("Requesting content ID: [" + req.params.id + "]"); - var source = connection.client.download({ - container: config.content_container(), - remote: encodeURIComponent(req.params.id) - }); + async.waterfall([ + async.apply(download_content, req.params.id), + inject_asset_vars + ], function (err, envelope) { + if (err) { + log.error("Failed to retrieve a metadata envelope", err); - // This directly sends the response to the caller, long term we'll - // probably not want to do this, but it allows the prototype to get functional - // - source.pipe(res); + res.status(err.statusCode || 500); + res.send(); + next(); - next(); + return; + } + + res.status(200); + res.json(envelope); + next(); + }); }; /** diff --git a/src/routes.js b/src/routes.js index e729bbe..2b0a133 100644 --- a/src/routes.js +++ b/src/routes.js @@ -13,5 +13,5 @@ exports.loadRoutes = function (server) { server.put('/content/:id', content.store); server.del('/content/:id', content.delete); - server.post('/assets', restify.bodyParser(), assets.accept); + server.post('/assets', restify.bodyParser(), restify.queryParser(), assets.accept); };