Hugo as AWS Lambda function does not copy static folder

I’m running this hugo file as my Amazon S3 lambda function, but for some reason it’s not copying the static folder from my input to output bucket on Amazon. I have checked the configuration and policies for both buckets, and they are correct–at least I followed the protocol mentioned in this article: http://bezdelev.com/post/hugo-aws-lambda-static-website/.

When I configure a test run on Amazon for a single static file, this hugo file will successfully copy that file to my output bucket. But it won’t copy the static folder as a whole on its own, as is the intent with AWS lambda functions. Any thoughts on how I might be able to change my RunHugo.js code below to successfully copy the static directory from my input to output bucket on AWS, as intended?

var async = require('async');
var util = require('util');
var spawn = require('child_process').spawn;
var s3 = require('s3');
var AWS = require('aws-sdk');

var syncClient = s3.createClient({
    maxAsyncS3: 20,
});

tmpDir = "/tmp/sources";
pubDir = tmpDir + "/public";

function isDir(dirName) {
    var isDirRe = new RegExp(/\/$/);
    return dirName.match(isDirRe) !== null;
}

function isStatic(fName) {
    var isStaticRe = new RegExp(/(^static\/|\/static\/)/);
    return fName.match(isStaticRe) !== null;
}

function staticFile(srcBucket, srcKey, dstBucket, context) {
    var awsS3 = new AWS.S3();
    var dst = srcKey.substring(7);
    var keyMatch = srcKey.match(/\/static\//);
    if (keyMatch !== null) {
        console.log("Key " + srcKey + " is in a theme content directory, removing prefix");
        dst = srcKey.substring(keyMatch.index + 8);
    }
    console.log("Dest: " + dst);
    awsS3.headObject({
        Bucket: srcBucket,
        Key: srcKey
    }, function(err, data) {
        if (err) { // an error occurred
            console.log("Obj not found");
            context.done(err);
        }
        awsS3.copyObject({
            ACL: 'public-read',
            Bucket: dstBucket,
            Key: dst,
            CopySource: srcBucket + '/' + srcKey,
            CopySourceIfNoneMatch: '"f482b82df157bee673b36145f9641005"',
            StorageClass: 'REDUCED_REDUNDANCY',
        }, function(err, data){
            if (err) { // an error occurred
                if (err.code === "PreconditionFailed") {
                    console.log("Static object already exists in S3 and is unmodified.")
                    context.done();
                } else {
                    console.log(util.inspect(err, {depth: 5}));
                    console.log(err.stack);
                    context.done(err);
                }
            } else {
                context.done();
            }
        });
    });
}

function siteGenerate(srcBucket, srcKey, dstBucket, context) {
    async.waterfall([
    function download(next) {
        var params = {
            localDir: tmpDir,
            s3Params: {
                Bucket: srcBucket,
            },
            getS3Params: function(localfile, s3Object, callback) {
                // skip static content
                if (isStatic(s3Object.Key)) {
                    callback(null, null);
                    return;
                }
                if (isDir(s3Object.Key)) {
                    callback(null, null);
                    return;
                }
                callback(null, {});
            },
        };
        var downloader = syncClient.downloadDir(params);
        downloader.on('error', function(err) {
            console.error("unable to sync down:", err.stack);
            next(err);
        });
        downloader.on('end', function() {
            console.log("done downloading");
            next(null);
        });
    },
    function staticDir(next) {
        var child = spawn("mkdir", ["-p", tmpDir + "/static"], {});
        child.on('error', function(err) {
            console.log("failed to make static dir: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("made static dir: " + code);
            next(null);
        });
    },
    function runHugo(next) {
        console.log("Running hugo");
        var child = spawn("./hugo", ["-v", "--source=" + tmpDir, "--destination=" + pubDir], {});
        child.stdout.on('data', function (data) {
            console.log('hugo-stdout: ' + data);
        });
        child.stderr.on('data', function (data) {
            console.log('hugo-stderr: ' + data);
        });
        child.on('error', function(err) {
            console.log("hugo failed with error: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("hugo exited with code: " + code);
            next(null);
        });
    },
    function upload(next) {
        var params = {
            localDir: pubDir,
            deleteRemoved: true,
            s3Params: {
                ACL: 'public-read',
                Bucket: dstBucket,
            },
        };
        var uploader = syncClient.uploadDir(params);
        uploader.on('error', function(err) {
            console.error("unable to sync up:", err.stack);
            next(err);
        });
        uploader.on('end', function() {
            console.log("done uploading");
            next(null);
        });
    },
    ], function(err) {
        if (err) console.error("Failure because of: " + err)
        else console.log("All methods in waterfall succeeded.");
        context.done();
    });
}

function handleFile(srcBucket, srcKey, dstBucket, context) {
    // bail for .git files
    if (srcKey.match(/^\.git\//) !== null) {
        context.done();
        return;
    }
    if (isDir(srcKey)) {
        context.done();
        return;
    }
    if (isStatic(srcKey)) {
        staticFile(srcBucket, srcKey, dstBucket, context);
    } else {
        siteGenerate(srcBucket, srcKey, dstBucket, context);
    }
}

exports.handler = function(event, context) {
    // Read options from the event.
    console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    var srcKey    = event.Records[0].s3.object.key;
    var dstBucket = event.Records[0].s3.bucket.name.replace('-in', '');

    console.log("Uploading to " + dstBucket + " bucket");

    // don't run hugo for git files
    if (srcKey.match(/^\.git\//) !== null) {
        console.log("Key " + srcKey + " is static content, bailing out");
        context.done();
    }

    handleFile(srcBucket, srcKey, dstBucket, context);
};
1 Like

I’m having the same issue. None of the static files are being copied to the output location. Have you found any solutions yet?

Twitter: @tfisher

Have you thought about letting Hugo do that for you by running Hugo from within your lambda function? That’s what I’m doing here and it’s working just fine.

I know its a really late update - its just the post I found whilst following the same path as the OP - so just in case anyone has the same issue …

After a suprising amount of frustration I ended up hacking the RunHugo.js file and whilst I was at it updated the version to 25-1 . Then I rezipped it and it works.

An appaling hack but it was not something I really wanted to work on - it was just for some testing following the same posting as the OP

Basically I added --forceSyncStatic to the hugo flags and then removed the code that originally prevented /static from copying over - the updated function is below . Don’t use this code if you have a lot of static data as its horrendously inefficient - but for small sites it should be OK

The function just has the hugo flags changed and

// skip static content
if (isStatic(s3Object.Key)) {
callback(null, null);
return;
}

code section removed

RunHugo.js

var async = require('async');
var util = require('util');
var spawn = require('child_process').spawn;
var s3 = require('s3');
var AWS = require('aws-sdk');

var syncClient = s3.createClient({
    maxAsyncS3: 20,
});

tmpDir = "/tmp/sources";
pubDir = tmpDir + "/public";

function isDir(dirName) {
    var isDirRe = new RegExp(/\/$/);
    return dirName.match(isDirRe) !== null;
}

function isStatic(fName) {
    var isStaticRe = new RegExp(/(^static\/|\/static\/)/);
    return fName.match(isStaticRe) !== null;
}

function staticFile(srcBucket, srcKey, dstBucket, context) {
    var awsS3 = new AWS.S3();
    var dst = srcKey.substring(7);
    var keyMatch = srcKey.match(/\/static\//);
    if (keyMatch !== null) {
        console.log("Key " + srcKey + " is in a theme content directory, removing prefix");
        dst = srcKey.substring(keyMatch.index + 8);
    }
    console.log("Dest: " + dst);
    awsS3.headObject({
        Bucket: srcBucket,
        Key: srcKey,
    }, function(err, data) {
        if (err) { // an error occurred
            console.log("Obj not found");
            context.done(err);
        }
        console.log("Obj has etag: " + data.ETag);
        awsS3.copyObject({
            ACL: 'public-read',
            Bucket: dstBucket,
            Key: dst,
            CopySource: srcBucket + '/' + srcKey,
            CopySourceIfNoneMatch: '"f482b82df157bee673b36145f9641005"',
            StorageClass: 'REDUCED_REDUNDANCY',
        }, function(err, data){

            if (err) { // an error occurred
                if (err.code === "PreconditionFailed") {
                    console.log("Static object already exists in S3 and is unmodified.")
                    context.done();
                } else {
                    console.log(util.inspect(err, {depth: 5}));
                    console.log(err.stack);
                    context.done(err);
                }
            } else {
                context.done();
            }
        });
    });
}

function siteGenerate(srcBucket, srcKey, dstBucket, context) {
    async.waterfall([
    function download(next) {
        var params = {
            localDir: tmpDir,

            s3Params: {
                Bucket: srcBucket,
            },
            getS3Params: function(localfile, s3Object, callback) {
                if (isDir(s3Object.Key)) {
                    callback(null, null);
                    return;
                }
                callback(null, {});
            },
        };
        var downloader = syncClient.downloadDir(params);
        downloader.on('error', function(err) {
            console.error("unable to sync down:", err.stack);
            next(err);
        });
        downloader.on('end', function() {
            console.log("done downloading");
            next(null);
        });
    },
    function staticDir(next) {
        var child = spawn("mkdir", ["-p", tmpDir + "/static"], {});
        child.on('error', function(err) {
            console.log("failed to make static dir: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("made static dir: " + code);
            next(null);
        });
    },
    function runHugo(next) {
        console.log("Running hugo");
        var child = spawn("./hugo", ["-v", "--forceSyncStatic=true","--source=" + tmpDir, "--destination=" + pubDir], {});
        child.stdout.on('data', function (data) {
            console.log('hugo-stdout: ' + data);
        });
        child.stderr.on('data', function (data) {
            console.log('hugo-stderr: ' + data);
        });
        child.on('error', function(err) {
            console.log("hugo failed with error: " + err);
            next(err);
        });
        child.on('close', function(code) {
            console.log("hugo exited with code: " + code);
            next(null);
        });
    },
    function upload(next) {
        var params = {
            localDir: pubDir,
            deleteRemoved: true,
            s3Params: {
                ACL: 'public-read',
                Bucket: dstBucket,
            },
        };
        var uploader = syncClient.uploadDir(params);
        uploader.on('error', function(err) {
            console.error("unable to sync up:", err.stack);
            next(err);
        });
        uploader.on('end', function() {
            console.log("done uploading");
            next(null);
        });
    },
    ], function(err) {
        if (err) console.error("Failure because of: " + err)
        else console.log("All methods in waterfall succeeded.");

        context.done();
    });
}

function handleFile(srcBucket, srcKey, dstBucket, context) {
    // bail for .git files
    if (srcKey.match(/^\.git\//) !== null) {
        context.done();
        return;
    }
    if (isDir(srcKey)) {
        context.done();
        return;
    }
    if (isStatic(srcKey)) {
        staticFile(srcBucket, srcKey, dstBucket, context);
    } else {
        siteGenerate(srcBucket, srcKey, dstBucket, context);
    }
}

exports.handler = function(event, context) {
    // Read options from the event.
    console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    var srcKey    = event.Records[0].s3.object.key;
    var dstBucket = event.Records[0].s3.bucket.name.replace('input.', '');

    // don't run hugo for git files
    if (srcKey.match(/^\.git\//) !== null) {
        console.log("Key " + srcKey + " is static content, bailing out");
        context.done();
    }
    handleFile(srcBucket, srcKey, dstBucket, context);
};