Implement extension loading api (#6378)

* Implement extension loading api

* Fix frontend tests

* Implement saving and unloading extensions

* Save after loading

* Add tests

* Fix coverage

* Refactor frontent test calls

* Update controller.js

* Review fixes

* Update utils.js

* Rename list topic

Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

lib/controller.js

@@ -7,9 +7,6 @@ const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
 const stringify = require('json-stable-stringify-without-jsonify');
-const fs = require('fs');
-const data = require('./util/data');
-const path = require('path');
 const assert = require('assert');
 
 // Extensions
@@ -30,12 +27,13 @@ const ExtensionReport = require('./extension/legacy/report');
 const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
 const ExtensionExternalConverters = require('./extension/externalConverters');
+const ExtensionExternalExtension = require('./extension/externalExtension');
 
 const AllExtensions = [
     ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
     ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
     ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
-    ExtensionExternalConverters, ExtensionFrontend,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension,
 ];
 
 class Controller {
@@ -50,6 +48,7 @@ class Controller {
         this.publishEntityState = this.publishEntityState.bind(this);
         this.enableDisableExtension = this.enableDisableExtension.bind(this);
         this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
+        this.addExtension = this.addExtension.bind(this);
 
         // Initialize extensions.
         const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
@@ -91,15 +90,7 @@ class Controller {
         if (settings.get().advanced.availability_timeout) {
             this.extensions.push(new ExtensionAvailability(...args));
         }
-
-        const extensionPath = data.joinPath('extension');
-        if (fs.existsSync(extensionPath)) {
-            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
-            for (const extension of extensions) {
-                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
-                this.extensions.push(new Extension(...args, settings, logger));
-            }
-        }
+        this.extensions.push(new ExtensionExternalExtension(...args, this.addExtension, this.enableDisableExtension));
     }
 
     async start() {
@@ -187,6 +178,10 @@ class Controller {
         }
     }
 
+    addExtension(extension) {
+        this.extensions.push(extension);
+    }
+
     async stop(reason=null) {
         // Call extensions
         await this.callExtensionMethod('stop', []);

lib/extension/externalExtension.js

@@ -0,0 +1,96 @@
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const fs = require('fs');
+const data = require('./../util/data');
+const path = require('path');
+const logger = require('./../util/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/extension/request/(.*)`);
+
+class ExternalExtension extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus, addExtension, enableDisableExtension) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.args = [zigbee, mqtt, state, publishEntityState, eventBus];
+        this.addExtension = addExtension;
+        this.enableDisableExtension = enableDisableExtension;
+        this.extensionsBaseDir = 'extension';
+        this.requestLookup = {
+            'save': this.saveExtension.bind(this),
+            'read': this.readExtensionCode.bind(this),
+        };
+        this.loadUserDefinedExtensions();
+    }
+    getExtensionsBasePath() {
+        return data.joinPath(this.extensionsBaseDir);
+    }
+
+    getListOfUserDefinedExtensions() {
+        const basePath = this.getExtensionsBasePath();
+        if (fs.existsSync(basePath)) {
+            return fs.readdirSync(basePath).filter((f) => f.endsWith('.js'));
+        } else {
+            return [];
+        }
+    }
+    saveExtension({name, content}) {
+        const ModuleConstructor = utils.loadModuleFromText(content);
+        this.loadExtension(ModuleConstructor);
+        const basePath = this.getExtensionsBasePath();
+        /* istanbul ignore else */
+        if (!fs.existsSync(basePath)) {
+            fs.mkdirSync(basePath);
+        }
+        const extensonFilePath = path.join(basePath, name);
+        fs.writeFileSync(extensonFilePath, content);
+        this.publishExtensions();
+        return utils.getResponse(`Extension ${name} loaded`, {}, null);
+    }
+    readExtensionCode({name}) {
+        const extensonFilePath = path.join(this.getExtensionsBasePath(), name);
+        const response = {name, content: fs.readFileSync(extensonFilePath, 'utf-8')};
+        return utils.getResponse(`Extension ${name} code read`, response, null);
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = utils.parseJSON(message, message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/extension/response/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/extension/response/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    loadExtension(ConstructorClass) {
+        this.enableDisableExtension(false, ConstructorClass.name);
+        this.addExtension(new ConstructorClass(...this.args, settings, logger));
+    }
+
+    loadUserDefinedExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        const extensionPath = this.getExtensionsBasePath();
+        for (const extension of extensions) {
+            const Extension = utils.loadModuleFromFile(path.join(extensionPath, extension));
+            this.loadExtension(Extension);
+        }
+    }
+    async onMQTTConnected() {
+        this.publishExtensions();
+    }
+
+    async publishExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        await this.mqtt.publish('bridge/extensions', stringify(extensions), {
+            retain: true,
+            qos: 0,
+        }, settings.get().mqtt.base_topic, true);
+    }
+}
+
+module.exports = ExternalExtension;

lib/util/utils.js

@@ -3,6 +3,7 @@ const humanizeDuration = require('humanize-duration');
 const data = require('./data');
 const vm = require('vm');
 const fs = require('fs');
+const path = require('path');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -144,6 +145,22 @@ function parseJSON(value, failedReturnValue) {
     }
 }
 
+function loadModuleFromText(moduleCode) {
+    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+    const sandbox = {
+        require: require,
+        module: {},
+        console,
+    };
+    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    return sandbox.module.exports;
+}
+
+function loadModuleFromFile(modulePath) {
+    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
+    return loadModuleFromText(moduleCode);
+}
+
 function* getExternalConvertersDefinitions(settings) {
     const externalConverters = settings.get().external_converters;
 
@@ -151,13 +168,7 @@ function* getExternalConvertersDefinitions(settings) {
         let converter;
 
         if (moduleName.endsWith('.js')) {
-            const sandbox = {
-                require,
-                module: {},
-            };
-            const converterCode = fs.readFileSync(data.joinPath(moduleName), {encoding: 'utf8'});
-            vm.runInNewContext(converterCode, sandbox);
-            converter = sandbox.module.exports;
+            converter = loadModuleFromFile(data.joinPath(moduleName));
         } else {
             converter = require(moduleName);
         }
@@ -237,4 +248,6 @@ module.exports = {
     parseJSON,
     getExternalConvertersDefinitions,
     validateFriendlyName,
+    loadModuleFromFile,
+    loadModuleFromText,
 };

test/assets/exampleExtension.js

@@ -1,6 +1,7 @@
 class Example {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.mqtt = mqtt;
+        this.mqtt.publish('example/extension', 'call from constructor')
     }
 
     onMQTTConnected() {

test/controller.test.js

@@ -603,16 +603,6 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
-    it('Load user extension', async () => {
-        const extensionPath = path.join(data.mockDir, 'extension');
-        fs.mkdirSync(extensionPath);
-        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
-    });
-
     it('Start controller with force_disable_retain', async () => {
         settings.set(['mqtt', 'force_disable_retain'], true);
         await controller.start();

test/externalExtension.test.js

@@ -0,0 +1,81 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = () => new Promise(setImmediate);
+const tmp = require('tmp');
+const mocksClear = [
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+const fs = require('fs');
+const mkdirSyncSpy = jest.spyOn(fs, 'mkdirSync');
+
+describe('User extensions', () => {
+    let controller;
+    let mockExit;
+
+    beforeEach(() => {
+        zigbeeHerdsman.returnDevices.splice(0);
+        mockExit = jest.fn();
+        controller = new Controller(jest.fn(), mockExit);
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
+    });
+    afterEach(() => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.rmdirSync(extensionPath, {recursive: true});
+    })
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify(['exampleExtension.js']), { retain: true, qos: 0 }, expect.any(Function));
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/extension/request/read', stringify({"name": "exampleExtension.js"}));
+        await flushPromises();
+        const expectedResponse = {"data": {"name": "exampleExtension.js", "content": extensionCode}, "status":"ok"};
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extension/response/read', stringify(expectedResponse), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('Load user extension from api call', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/extension/request/save', stringify({"name": "foo.js", "content": extensionCode}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify(['foo.js']), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'call from constructor', { retain: false, qos: 0 }, expect.any(Function));
+        expect(mkdirSyncSpy).toHaveBeenCalledWith(extensionPath);
+    });
+
+    it('Do not load corrupted extensions', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = "definetly not a correct javascript code";
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/extension/request/save', stringify({"name": "foo.js", "content": extensionCode}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extension/response/save', stringify({"data":{},"error":"Unexpected identifier","status":"error"}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+});

test/frontend.test.js

@@ -119,8 +119,8 @@ describe('Frontend', () => {
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await mockWS.events.connection(mockWSClient.implementation);
 
-        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[0])).toStrictEqual({topic: 'bridge/state', payload: 'online'});
-        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}});
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bridge/state', payload: 'online'}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}}));
 
         // Message
         MQTT.publish.mockClear();
@@ -155,7 +155,7 @@ describe('Frontend', () => {
         settings.set(['advanced'], {last_seen: 'ISO_8601'});
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await mockWS.events.connection(mockWSClient.implementation);
-        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}});
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}}));
     });
 
     it('onReques/onUpgrade', async () => {