About tasks folder and executable files

[loretoparisi]

I don't get this code

.then(function(result) {
    return execute(result, {
      shell: "cp /var/task/ffmpeg /tmp/.; chmod 755 /tmp/ffmpeg", // copies an ffmpeg binary to /tmp/ and chmods permissions to run it
      logOutput: true
    })
  })

Is that means that the executable must be in the /var/task folder or they can be in a arbitrary folder?
When I upload my zip file (via S3) to Lambda the executable are at the same level of the index.js like

macbookproloreto:src admin$ tree -L 1
.
├── config.js
├── ffmpeg
├── ffprobe
├── gulpfile.js
├── index.js
└── lambdaapihelper.js

So in this case can I do like

.then(function(result) {
    return execute(result, {
      shell: "cp ./ffmpeg /tmp/.; chmod 755 /tmp/ffmpeg", // copies an ffmpeg binary to /tmp/ and chmods permissions to run it
      logOutput: true
    })
  })

and I get as logging

2016-06-15T13:02:26.119Z    685476e8-32f9-11e6-b004-e5e236444b4c    exec stdout: ffmpeg fp_a8dba864e1571d39e786389317e5619a_1.mp4  
2016-06-15T13:02:26.120Z    685476e8-32f9-11e6-b004-e5e236444b4c    exec stderr:  
2016-06-15T13:02:26.120Z    685476e8-32f9-11e6-b004-e5e236444b4c    spawning shell 
2016-06-15T13:02:26.120Z    685476e8-32f9-11e6-b004-e5e236444b4c    { shell: 'cp /var/task/ffmpeg /tmp/.; chmod 755 /tmp/ffmpeg', logOutput: true }

Is that correct? This brings me to the second question about logging, where is logging output going? I can see in CloudWatch

START RequestId: 64aec75d-32f9-11e6-889b-6381565ec002 Version: $LATEST 
2016-06-15T13:02:21.821Z    64aec75d-32f9-11e6-889b-6381565ec002    spawning shell 
2016-06-15T13:02:21.822Z    64aec75d-32f9-11e6-889b-6381565ec002    { shell: 'echo `ls /tmp/`', logOutput: true }
2016-06-15T13:02:26.981Z    685476e8-32f9-11e6-b004-e5e236444b4c    exec stdout:  
2016-06-15T13:02:26.981Z    685476e8-32f9-11e6-b004-e5e236444b4c    exec stderr: 

but not the commands output logged for the ls.

Latest but not least question, as soon as I configure the executable (hopefully ok) I call this function

MediaHelper.prototype.probe = function(fpath) {
      var self=this;
      return new Promise((resolve, reject) => {
            const args = [
                '-v', 'quiet',
                '-print_format', 'json',
                '-show_format',
          '-show_streams',
                '-i', fpath
            ];
            const opts = {
                cwd: self._options.tempDir
            };
            const cb = (error, stdout) => {
                if (error)
                    return reject(error);
          try {
            const outputObj = JSON.parse(stdout);
            return resolve( outputObj );
          } catch(ex) {
              return reject(ex);
          }
        };
        cp.execFile('ffprobe', args, opts, cb)
                .on('error', reject);
        });
    }//probe

and this one for ffmpeg

MediaHelper.prototype.cut = function(fpath,opath,duration,start) {
      var self=this;

      // defaults
      start = start || '00:00:00';//start 0
      duration = duration || self._options.maxDuration+''; //default to maxDuration

      return new Promise((resolve, reject) => {
            const args = [
          '-y',
                '-loglevel', 'warning',
                '-i', fpath,
          '-ss', start,
          '-t', duration,
          '-async', '1',
          opath
            ];
            const opts = {
                cwd: self._options.tempDir
            };
        cp.spawn('ffmpeg', args, opts)
                .on('message', msg => self.logger.info(msg))
                .on('error', reject)
                .on('close', resolve);
        });
    }//cut

what I get is this then

2016-06-15T13:02:26.119Z    685476e8-32f9-11e6-b004-e5e236444b4c    exec stdout: ffmpeg fp_a8dba864e1571d39e786389317e5619a_1.mp4  
2016-06-15T13:02:26.120Z    685476e8-32f9-11e6-b004-e5e236444b4c    exec stderr: 

Does this mean that the executable was not found? While the lines does not show exec arguments I have passed to cp.execFile?

Thank you.

[loretoparisi]

[UPDATE]

This is my complete env setup function

Configure : function(logger) {

        this.logger=logger;
        var self=this;

        // add temporary folder and task root folder to PATH
        process.env['PATH'] = process.env['PATH'] + ':/tmp/:' + process.env['LAMBDA_TASK_ROOT']

        var result = {}
         execute(result, {
           shell: "echo `ls /tmp/`", // logs output of /tmp/ dir on your lambda machine
           logOutput: true
         })
         .then(function(result) {
           self.logger.info("MediaHelper tmp ls %s", result);
           return execute(result, {
             shell: "cp ./ffmpeg /tmp/.; chmod 755 /tmp/ffmpeg", // copies an ffmpeg binary to /tmp/ and chmods permissions to run it
             logOutput: true
           })
         })
         .then(function(result) {
           self.logger.info("MediaHelper ffmpeg copy %s", result);
           return execute(result, {
             shell: "cp ./ffprobe /tmp/.; chmod 755 /tmp/ffprobe", // copies an ffmpeg binary to /tmp/ and chmods permissions to run it
             logOutput: true
           })
         })
         .then(function(result) {
           elf.logger.info("MediaHelper ffprobe copy %s", result);
           return execute(result, {
             shell: "echo `ls /tmp/`", // logs output of /tmp/ dir on your lambda machine
             logOutput: true
           })
         })
         .then(function(result) {
           self.logger.info("MediaHelper tmp ls %s", result);
        })
         .fail(function(err) {
          self.logger.error("MediaHelper: Failed executable setup %s",err);
        });
      }

for the folder structure showed above.

[loretoparisi]

[UPDATE]
I confirm that this solution worked i.e. it was a path issue. Next big question is, avoid to copy executable at every execution from ./ to /tmp/ folder. An idea could be to copy those files in /var/task folder the LAMBDA_TASK_ROOT, where those binary files should persist at every execution...

[russmatney]

Thanks for bringing this up - i need to update the example, i don't believe
the copy step is necessary anymore. I'm on vacation now but will take a
full look when i'm back next week.

It may be the same detail addressed in the update here:
http://stackoverflow.com/a/29001078/860787

This was a year ago now - it may be that lamba is even easier to
use/abstract by this point. I'm definitely open to any execution related
ideas that this module could handle.
On Wed, Jun 15, 2016 at 3:17 PM Loreto Parisi [email protected]
wrote:

[UPDATE]
I confirm that this solution worked i.e. it was a path issue. Next big
question is, avoid to copy executable at every execution, is this something
that your module could handle?

—
You are receiving this because you are subscribed to this thread.
Reply to this email directly, view it on GitHub
#2 (comment),
or mute the thread
https://github.com/notifications/unsubscribe/ABhbvpcCzXByWtWf3eAeCsPgSATiM3lhks5qMAlpgaJpZM4I2VD8
.

[loretoparisi]

@russmatney Thanks, what I have found is that without the copy the chmod was not working in the var/task (that is pwd at the end) dir. Then I think we still have to cp and chmod from the /tmp folder. I have read the reply, and I will try again, what I have got from CloudWatch was a FORBIDDEN error...

[russmatney]

Hey @loretoparisi – sorry for the delay on my part here. I've been catching up with everything and have finally made time for this.

It's been over a year since I've worked with aws lambda directly, I'm happy to re-familiarize and clean this up if it's of benefit to you (and probably others).

From what I understand, it seems a simple solution would be another api option that handles the cp and chmod behind the scenes, making it simple for the user to expose binaries (like ffmpeg) for reference by future commands/scripts. Is that the kind of thing you're looking for?

Just want to make sure I'm on the right track here. If it'd be more helpful for me to focus on the other questions you had re: logging, etc. Perhaps if there's enough meat there, I'll start an outline for a wiki with some practical usage details.

Let me know and I can get a prototype going!

[loretoparisi]

Hi @russmatney, so at the end I realized that, since recents lambda OS udpates, there is not need anymore to cp binaries elsewhere. This now is simply as copying ffmpeg and ffprobe binaries in the dist/ folder (or wherever you prefer) that you are going to distribute to lambda backend. Before that, just chmod 755 ./ff*before uploading, that was exactly the same permission change we were doing on lambda before so I have made this optional on the lambda environment now

if(self._options.copyBinaries)
        {
          var result = {}
          execute(result, {
            shell: "cp ./ffmpeg /tmp/.; chmod 755 /tmp/ffmpeg", // copies an ffmpeg binary to /tmp/ and chmods permissions to run it
            logOutput: true
          })
          .then(function(result) {
            return execute(result, {
               shell: "cp ./ffprobe /tmp/.; chmod 755 /tmp/ffprobe", // copies an ffmpeg binary to /tmp/ and chmods permissions to run it
               logOutput: true
             })
          })
          .then(function(result) {
             self.logger.info("LambdaAPIHelper.Configure done.");
          })
          .fail(function(err) {
            self.logger.error("LambdaAPIHelper.Configure: error %s",err);
          });
        } //copyBinaries

but having self._options.copyBinaries=false since so far it does work without this copy phase.
So, at the end, in the lambda I just have the same tree in my LAMBDA_TASK_ROOT folder:

├── config.js
├── ffmpeg
├── ffprobe
├── index.js
└── lambdaapihelper.js

and I just execute those binaries using as path ./ with the same scripts I pasted here for the MediaHelper class.

Thank you, and if you make a wiki for logging question, etc. this would help as well!