Hugo as AWS Lambda function does not copy static folder

support
#1

I’m running this hugo file as my Amazon S3 lambda function, but for some reason it’s not copying the static folder from my input to output bucket on Amazon. I have checked the configuration and policies for both buckets, and they are correct–at least I followed the protocol mentioned in this article: http://bezdelev.com/post/hugo-aws-lambda-static-website/.

When I configure a test run on Amazon for a single static file, this hugo file will successfully copy that file to my output bucket. But it won’t copy the static folder as a whole on its own, as is the intent with AWS lambda functions. Any thoughts on how I might be able to change my RunHugo.js code below to successfully copy the static directory from my input to output bucket on AWS, as intended?

var async = require('async');
var util = require('util');
var spawn = require('child_process').spawn;
var s3 = require('s3');
var AWS = require('aws-sdk');

var syncClient = s3.createClient({
    maxAsyncS3: 20,
});

tmpDir = "/tmp/sources";
pubDir = tmpDir + "/public";

function isDir(dirName) {
    var isDirRe = new RegExp(/\/$/);
    return dirName.match(isDirRe) !== null;
}

function isStatic(fName) {
    var isStaticRe = new RegExp(/(^static\/|\/static\/)/);
    return fName.match(isStaticRe) !== null;
}

function staticFile(srcBucket, srcKey, dstBucket, context) {
    var awsS3 = new AWS.S3();
    var dst = srcKey.substring(7);
    var keyMatch = srcKey.match(/\/static\//);
    if (keyMatch !== null) {
        console.log("Key " + srcKey + " is in a theme content directory, removing prefix");
        dst = srcKey.substring(keyMatch.index + 8);
    }
    console.log("Dest: " + dst);
    awsS3.headObject({
        Bucket: srcBucket,
        Key: srcKey
    }, function(err, data) {
        if (err) { // an error occurred
            console.log("Obj not found");
            context.done(err);
        }
        awsS3.copyObject({
            ACL: 'public-read',
            Bucket: dstBucket,
            Key: dst,
            CopySource: srcBucket + '/' + srcKey,
            CopySourceIfNoneMatch: '"f482b82df157bee673b36145f9641005"',
            StorageClass: 'REDUCED_REDUNDANCY',
        }, function(err, data){
            if (err) { // an error occurred
                if (err.code === "PreconditionFailed") {
                    console.log("Static object already exists in S3 and is unmodified.")
                    context.done();
                } else {
                    console.log(util.inspect(err, {depth: 5}));
                    console.log(err.stack);
                    context.done(err);
                }
            } else {
                context.done();
            }
        });
    });
}

function siteGenerate(srcBucket, srcKey, dstBucket, context) {
    async.waterfall([
    function download(next) {
        var params = {
            localDir: tmpDir,
            s3Params: {
                Bucket: srcBucket,
            },
            getS3Params: function(localfile, s3Object, callback) {
                // skip static content
                if (isStatic(s3Object.Key)) {
                    callback(null, null);
                    return;
                }
                if (isDir(s3Object.Key)) {
                    callback(null, null);
                    return;
                }
                callback(null, {});
            },
        };
        var downloader = syncClient.downloadDir(params);
        downloader.on('error', function(err) {
            console.error("unable to sync down:", err.stack);
            next(err);
        });
        downloader.on('end', function() {
            console.log("done downloading");
            next(null);
        });
    },
    function staticDir(next) {
        var child = spawn("mkdir", ["-p", tmpDir + "/static"], {});
        child.on('error', function(err) {
            console.log("failed to make static dir: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("made static dir: " + code);
            next(null);
        });
    },
    function runHugo(next) {
        console.log("Running hugo");
        var child = spawn("./hugo", ["-v", "--source=" + tmpDir, "--destination=" + pubDir], {});
        child.stdout.on('data', function (data) {
            console.log('hugo-stdout: ' + data);
        });
        child.stderr.on('data', function (data) {
            console.log('hugo-stderr: ' + data);
        });
        child.on('error', function(err) {
            console.log("hugo failed with error: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("hugo exited with code: " + code);
            next(null);
        });
    },
    function upload(next) {
        var params = {
            localDir: pubDir,
            deleteRemoved: true,
            s3Params: {
                ACL: 'public-read',
                Bucket: dstBucket,
            },
        };
        var uploader = syncClient.uploadDir(params);
        uploader.on('error', function(err) {
            console.error("unable to sync up:", err.stack);
            next(err);
        });
        uploader.on('end', function() {
            console.log("done uploading");
            next(null);
        });
    },
    ], function(err) {
        if (err) console.error("Failure because of: " + err)
        else console.log("All methods in waterfall succeeded.");
        context.done();
    });
}

function handleFile(srcBucket, srcKey, dstBucket, context) {
    // bail for .git files
    if (srcKey.match(/^\.git\//) !== null) {
        context.done();
        return;
    }
    if (isDir(srcKey)) {
        context.done();
        return;
    }
    if (isStatic(srcKey)) {
        staticFile(srcBucket, srcKey, dstBucket, context);
    } else {
        siteGenerate(srcBucket, srcKey, dstBucket, context);
    }
}

exports.handler = function(event, context) {
    // Read options from the event.
    console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    var srcKey    = event.Records[0].s3.object.key;
    var dstBucket = event.Records[0].s3.bucket.name.replace('-in', '');

    console.log("Uploading to " + dstBucket + " bucket");

    // don't run hugo for git files
    if (srcKey.match(/^\.git\//) !== null) {
        console.log("Key " + srcKey + " is static content, bailing out");
        context.done();
    }

    handleFile(srcBucket, srcKey, dstBucket, context);
};
1 Like
#2

I’m having the same issue. None of the static files are being copied to the output location. Have you found any solutions yet?

Twitter: @tfisher

#3

Have you thought about letting Hugo do that for you by running Hugo from within your lambda function? That’s what I’m doing here and it’s working just fine.

#4

I know its a really late update - its just the post I found whilst following the same path as the OP - so just in case anyone has the same issue …

After a suprising amount of frustration I ended up hacking the RunHugo.js file and whilst I was at it updated the version to 25-1 . Then I rezipped it and it works.

An appaling hack but it was not something I really wanted to work on - it was just for some testing following the same posting as the OP

Basically I added --forceSyncStatic to the hugo flags and then removed the code that originally prevented /static from copying over - the updated function is below . Don’t use this code if you have a lot of static data as its horrendously inefficient - but for small sites it should be OK

The function just has the hugo flags changed and

// skip static content
if (isStatic(s3Object.Key)) {
callback(null, null);
return;
}

code section removed

RunHugo.js

var async = require('async');
var util = require('util');
var spawn = require('child_process').spawn;
var s3 = require('s3');
var AWS = require('aws-sdk');

var syncClient = s3.createClient({
    maxAsyncS3: 20,
});

tmpDir = "/tmp/sources";
pubDir = tmpDir + "/public";

function isDir(dirName) {
    var isDirRe = new RegExp(/\/$/);
    return dirName.match(isDirRe) !== null;
}

function isStatic(fName) {
    var isStaticRe = new RegExp(/(^static\/|\/static\/)/);
    return fName.match(isStaticRe) !== null;
}

function staticFile(srcBucket, srcKey, dstBucket, context) {
    var awsS3 = new AWS.S3();
    var dst = srcKey.substring(7);
    var keyMatch = srcKey.match(/\/static\//);
    if (keyMatch !== null) {
        console.log("Key " + srcKey + " is in a theme content directory, removing prefix");
        dst = srcKey.substring(keyMatch.index + 8);
    }
    console.log("Dest: " + dst);
    awsS3.headObject({
        Bucket: srcBucket,
        Key: srcKey,
    }, function(err, data) {
        if (err) { // an error occurred
            console.log("Obj not found");
            context.done(err);
        }
        console.log("Obj has etag: " + data.ETag);
        awsS3.copyObject({
            ACL: 'public-read',
            Bucket: dstBucket,
            Key: dst,
            CopySource: srcBucket + '/' + srcKey,
            CopySourceIfNoneMatch: '"f482b82df157bee673b36145f9641005"',
            StorageClass: 'REDUCED_REDUNDANCY',
        }, function(err, data){

            if (err) { // an error occurred
                if (err.code === "PreconditionFailed") {
                    console.log("Static object already exists in S3 and is unmodified.")
                    context.done();
                } else {
                    console.log(util.inspect(err, {depth: 5}));
                    console.log(err.stack);
                    context.done(err);
                }
            } else {
                context.done();
            }
        });
    });
}

function siteGenerate(srcBucket, srcKey, dstBucket, context) {
    async.waterfall([
    function download(next) {
        var params = {
            localDir: tmpDir,

            s3Params: {
                Bucket: srcBucket,
            },
            getS3Params: function(localfile, s3Object, callback) {
                if (isDir(s3Object.Key)) {
                    callback(null, null);
                    return;
                }
                callback(null, {});
            },
        };
        var downloader = syncClient.downloadDir(params);
        downloader.on('error', function(err) {
            console.error("unable to sync down:", err.stack);
            next(err);
        });
        downloader.on('end', function() {
            console.log("done downloading");
            next(null);
        });
    },
    function staticDir(next) {
        var child = spawn("mkdir", ["-p", tmpDir + "/static"], {});
        child.on('error', function(err) {
            console.log("failed to make static dir: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("made static dir: " + code);
            next(null);
        });
    },
    function runHugo(next) {
        console.log("Running hugo");
        var child = spawn("./hugo", ["-v", "--forceSyncStatic=true","--source=" + tmpDir, "--destination=" + pubDir], {});
        child.stdout.on('data', function (data) {
            console.log('hugo-stdout: ' + data);
        });
        child.stderr.on('data', function (data) {
            console.log('hugo-stderr: ' + data);
        });
        child.on('error', function(err) {
            console.log("hugo failed with error: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("hugo exited with code: " + code);
            next(null);
        });
    },
    function upload(next) {
        var params = {
            localDir: pubDir,
            deleteRemoved: true,
            s3Params: {
                ACL: 'public-read',
                Bucket: dstBucket,
            },
        };
        var uploader = syncClient.uploadDir(params);
        uploader.on('error', function(err) {
            console.error("unable to sync up:", err.stack);
            next(err);
        });
        uploader.on('end', function() {
            console.log("done uploading");
            next(null);
        });
    },
    ], function(err) {
        if (err) console.error("Failure because of: " + err)
        else console.log("All methods in waterfall succeeded.");

        context.done();
    });
}

function handleFile(srcBucket, srcKey, dstBucket, context) {
    // bail for .git files
    if (srcKey.match(/^\.git\//) !== null) {
        context.done();
        return;
    }
    if (isDir(srcKey)) {
        context.done();
        return;
    }
    if (isStatic(srcKey)) {
        staticFile(srcBucket, srcKey, dstBucket, context);
    } else {
        siteGenerate(srcBucket, srcKey, dstBucket, context);
    }
}

exports.handler = function(event, context) {
    // Read options from the event.
    console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    var srcKey    = event.Records[0].s3.object.key;
    var dstBucket = event.Records[0].s3.bucket.name.replace('input.', '');

    // don't run hugo for git files
    if (srcKey.match(/^\.git\//) !== null) {
        console.log("Key " + srcKey + " is static content, bailing out");
        context.done();
    }
    handleFile(srcBucket, srcKey, dstBucket, context);
};