Di/ios provider/file and logs/qa 16207

lib/units/ios-device/support/storage.js → lib/units/base-device/support/storage.js

@@ -6,12 +6,15 @@ import request from 'postman-request'
 import logger from '../../../util/logger.js'
 export default syrup.serial()
     .define(options => {
-        const log = logger.createLogger('ios-device:support:storage')
+        const log = logger.createLogger('evice:support:storage')
         let plugin = Object.create(null)
         plugin.store = function(type, stream, meta) {
             let resolver = Promise.defer()
             let args = {
                 url: url.resolve(options.storageUrl, util.format('s/upload/%s', type))
+                , headers: {
+                    internal: 'Internal ' + meta.jwt
+                }
             }
             let req = request.post(args, function(err, res, body) {
                 if (err) {

lib/units/device/plugins/filesystem.js

@@ -6,7 +6,7 @@ import wireutil from '../../../wire/util.js'
 import adb from '../support/adb.js'
 import router from '../../base-device/support/router.js'
 import push from '../../base-device/support/push.js'
-import storage from '../support/storage.js'
+import storage from '../../base-device/support/storage.js'
 export default syrup.serial()
     .dependency(adb)
     .dependency(router)

lib/units/device/plugins/screen/capture.js

@@ -7,7 +7,7 @@ import wireutil from '../../../../wire/util.js'
 import adb from '../../support/adb.js'
 import router from '../../../base-device/support/router.js'
 import push from '../../../base-device/support/push.js'
-import storage from '../../support/storage.js'
+import storage from '../../../base-device/support/storage.js'
 import minicap from '../../resources/minicap.js'
 import display from '../util/display.js'
 export default syrup.serial()

lib/units/ios-device/index.js

@@ -9,13 +9,14 @@ import wda from './plugins/wda.js'
 import push from '../base-device/support/push.js'
 import sub from '../base-device/support/sub.js'
 import group from './plugins/group.js'
-import storage from './support/storage.js'
+import storage from '../base-device/support/storage.js'
 import devicelog from './plugins/devicelog.js'
 import stream from './plugins/screen/stream.js'
 import install from './plugins/install.js'
 import reboot from './plugins/reboot.js'
 import clipboard from './plugins/clipboard.js'
 import remotedebug from './plugins/remotedebug.js'
+import filesystem from './plugins/filesystem.js'
 export default (function(options) {
     // Show serial number in logs
     logger.setGlobalIdentifier(options.serial)
@@ -39,6 +40,7 @@ export default (function(options) {
                 .dependency(reboot)
                 .dependency(clipboard)
                 .dependency(remotedebug)
+                .dependency(filesystem)
                 .define(function(options, heartbeat, solo, info, wda) {
                     if (process.send) {
                         process.send('ready')

lib/units/ios-device/plugins/devicelog.js

@@ -9,14 +9,14 @@ import push from '../../base-device/support/push.js'
 import router from '../../base-device/support/router.js'
 import sub from '../../base-device/support/sub.js'
 import group from './group.js'
+import {IncompleteJsonParser} from 'incomplete-json-parser'
 
-export default syrup
-    .serial()
+export default syrup.serial()
     .dependency(push)
     .dependency(router)
     .dependency(sub)
     .dependency(group)
-    .define(function(options, push, router, group) {
+    .define(function(options, push, router, sub, group) {
         const log = logger.createLogger('device:plugins:devicelog')
 
         let DeviceLogger = {
@@ -31,17 +31,74 @@ export default syrup
             , startLogging: async function(channel) {
                 DeviceLogger.setChannel(channel)
                 try {
-                    DeviceLogger.stream = spawn('idb logs --udid ' + options.serial, {shell: true})
+                    const realGroup = await group.get()
+                    const parser = new IncompleteJsonParser()
+
+                    DeviceLogger.stream = spawn('idb', ['log', '--udid', options.serial, '--', '--style', 'json'])
 
                     DeviceLogger.stream.stdout.on('data', data => {
-                        push.send([
-                            this.channel
-                            , wireutil.envelope(new wire.DeviceLogcatEntryMessage(options.serial, new Date().getTime() / 1000, DeviceLogger.stream.pid, DeviceLogger.stream.pid, 1, 'device:log:cat', data.toString()))
-                        ])
+                        parser.write(data.toString())
+                        let logs = parser.getObjects()
+                        let threadId = 0
+                        let processId = 0
+                        let subsystem = '*'
+                        let message
+                        logs.forEach(log => {
+                            let logLevel = 2
+                            switch (log.messageType) {
+                            case 'Fatal': { logLevel = 7; break }
+                            case 'Error': { logLevel = 6; break }
+                            case 'Warning': { logLevel = 5; break }
+                            case 'Default': { logLevel = 4; break }
+                            case 'Activity': { logLevel = 3; break }
+                            }
+
+                            if (log.eventMessage) {
+                                message = log.eventMessage
+                            }
+                            else if (log.formatString) {
+                                message = log.formatString
+                            }
+                            else {
+                                return
+                            }
+
+                            if (log.threadID) {
+                                threadId = log.threadID
+                            }
+
+                            if (log.processID) {
+                                processId = log.processID
+                            }
+
+                            if (log.subsystem) {
+                                subsystem = log.subsystem
+                            }
+                            else {
+                                subsystem = '*'
+                            }
+
+                            push.send([
+                                realGroup.group
+                                , wireutil.envelope(new wire.DeviceLogcatEntryMessage(
+                                    options.serial
+                                    , new Date().getTime() / 1000
+                                    , processId
+                                    , threadId
+                                    , logLevel
+                                    , subsystem
+                                    , message
+                                ))
+                            ])
+                        })
+                    })
+
+                    DeviceLogger.stream.stderr.on('data', data => {
+                        console.log(data.toString())
                     })
 
                     DeviceLogger.stream.on('close', err => {
-                        log.fatal('Unable to get devicelog', err)
+                        log.warn('Stream is closed', err)
                         DeviceLogger.killLoggingProcess()
                     })
                 }
@@ -52,7 +109,6 @@ export default syrup
 
             , killLoggingProcess: () => {
                 if (DeviceLogger.stream) {
-                    process.kill(-DeviceLogger.stream.pid)
                     DeviceLogger.stream.kill()
                     DeviceLogger.stream = null
                     DeviceLogger.channel = ''
@@ -76,7 +132,9 @@ export default syrup
                 }
             })
             .on(wire.LogcatStopMessage, function(channel, data) {
+                const reply = wireutil.reply(options.serial)
                 DeviceLogger.killLoggingProcess()
+                push.send([channel, reply.okay('success')])
             })
             .on(wire.GroupMessage, function(channel, data) {
                 DeviceLogger.channel = channel

lib/units/ios-device/plugins/filesystem.js

@@ -0,0 +1,141 @@
+import syrup from '@devicefarmer/stf-syrup'
+import logger from '../../../util/logger.js'
+import wire from '../../../wire/index.js'
+import wireutil from '../../../wire/util.js'
+import router from '../../base-device/support/router.js'
+import push from '../../base-device/support/push.js'
+import storage from '../../base-device/support/storage.js'
+import {exec} from 'child_process'
+import path from 'path'
+import fs from 'fs'
+export default syrup.serial()
+    .dependency(router)
+    .dependency(push)
+    .dependency(storage)
+    .define(function(options, router, push, storage) {
+        const log = logger.createLogger('device:plugins:filesystem')
+        let plugin = Object.create(null)
+
+        const getRootDir = dir => {
+            let currentPath = dir.split('/')
+            let rootDir = currentPath[1]
+            let target = 'media'
+            switch (rootDir) {
+            case 'root': {
+                // only for simulators
+                return 'root'
+            }
+            case 'application': {
+                return 'application'
+            }
+            case 'crashes': {
+                return 'crashes'
+            }
+            }
+            return target
+        }
+
+        router.on(wire.FileSystemGetMessage, function(channel, message) {
+            let reply = wireutil.reply(options.serial)
+            let file = message.file
+            log.info('Retrieving file "%s"', file)
+            let currentPath = file.split('/')
+            let target = getRootDir(file)
+
+            if (currentPath.length === 2) {
+                currentPath = ''
+            }
+            else {
+                currentPath = currentPath.slice(2).join('/').replace(' ', '\\ ')
+            }
+
+            exec(
+                `idb file pull --${target} ./${currentPath} /tmp/tmp123 --json --udid ${options.serial}`,
+                (error, stdout, stderr) => {
+                    if (error) {
+                        log.warn('Unable to list directory "%s"', file, stderr)
+                    }
+
+                    storage.store('blob', path.basename(file), {
+                        filename: path.basename(file)
+                        , contentType: 'application/octet-stream'
+                    }).then((file) => {
+                        try {
+                            push.send([
+                                channel
+                                , reply.okay('success', file)
+                            ])
+                        }
+                        catch (e) {
+                            push.send([
+                                channel
+                                , reply.fail('error', e)
+                            ])
+                        }
+                        fs.unlink('/tmp/tmp123', (err) => {
+                            log.warn('Error while deleting file "%s"', file, err)
+                        })
+                    })
+                }
+            )
+        })
+        router.on(wire.FileSystemListMessage, function(channel, message) {
+            let reply = wireutil.reply(options.serial)
+            let dirs = []
+            let rootDir = message.dir
+            if (rootDir === '/') {
+                dirs = [
+                    {name: 'root', mtime: '1970-01-01T00:00:00.000Z', atime: null, ctime: null, birthtime: null, mode: 16877, size: 0}
+                    , {name: 'application', mtime: '1970-01-01T00:00:00.000Z', atime: null, ctime: null, birthtime: null, mode: 16877, size: 0}
+                    , {name: 'crashes', mtime: '1970-01-01T00:00:00.000Z', atime: null, ctime: null, birthtime: null, mode: 16877, size: 0}
+                    , {name: 'media', mtime: '1970-01-01T00:00:00.000Z', atime: null, ctime: null, birthtime: null, mode: 16877, size: 0}
+                ]
+                push.send([
+                    channel
+                    , reply.okay('success', dirs)
+                ])
+            }
+            let currentPath = message.dir.split('/')
+            let target = getRootDir(message.dir)
+
+            if (currentPath.length === 2) {
+                currentPath = ''
+            }
+            else {
+                currentPath = currentPath.slice(2).join('/').replace(' ', '\\ ')
+            }
+
+            exec(
+                `idb file list --${target} ./${currentPath} --json --udid ${options.serial}`,
+                (error, stdout, stderr) => {
+                    if (error) {
+                        log.warn('Unable to list directory "%s"', message.dir, stderr)
+                        push.send([
+                            channel
+                            , reply.fail(error.message)
+                        ])
+                        return
+                    }
+
+                    let rawDirs = JSON.parse(stdout)
+                    let dirs = []
+                    rawDirs.forEach(dir => {
+                        let name = dir.path
+                        let mode = 16629
+                        if (name.includes('.')) {
+                            mode = 0o555
+                        }
+                        dirs.push({
+                            name: name, mtime: '1970-01-01T00:00:00.000Z', atime: null, ctime: null, birthtime: null, mode: mode, size: 0
+                        })
+                    })
+
+                    push.send([
+                        channel
+                        , reply.okay('success', dirs)
+                    ])
+                }
+            )
+        })
+        return plugin
+    })

lib/units/ios-device/plugins/group.js

@@ -26,7 +26,7 @@ export default syrup.serial()
         const baseUrl = iosutil.getUri(options.wdaHost, options.wdaPort)
         let currentGroup = null
         let plugin = new events.EventEmitter()
-        plugin.get = Promise.method(() => {
+        plugin.get = Promise.method(function() {
             if (!currentGroup) {
                 throw new grouputil.NoGroupError()
             }
@@ -193,9 +193,7 @@ export default syrup.serial()
             .on(wire.GroupMessage, (channel, message) => {
                 let reply = wireutil.reply(options.serial)
                 // grouputil.match(ident, message.requirements)
-                Promise.method(() => {
-                    return plugin.join(message.owner, message.timeout, message.usage)
-                })()
+                plugin.join(message.owner, message.timeout, message.usage)
                     .then(() => {
                         push.send([
                             channel