Add force_disable_retain option (#4948)

* support configurable retain on mqtt settings since some servers throw errors and drop the connections if you try (e.g. aws iot)

* Revert "support configurable retain on mqtt settings since some servers throw errors and drop the connections if you try (e.g. aws iot)"

This reverts commit 40d3a9c0bc84c83d2b167e2e3c3a06ca6df80f47.

* support configurable retain on mqtt settings

* support subscribing to the whole base prefix

* Update mqtt.js

* Fixes

* Fixes

* Fix tests

* Updates

Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

test/bridge.test.js

@@ -42,7 +42,7 @@ describe('Bridge', () => {
         const directory = settings.get().advanced.log_directory;
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/info',
-          stringify({"commit":version.commitHash,"config":{"advanced":{"adapter_concurrent":null,"availability_blacklist":[],"availability_blocklist":[],"availability_passlist":[],"availability_timeout":0,"availability_whitelist":[],"cache_state":true,"cache_state_persistent":true,"cache_state_send_on_startup":true,"channel":11,"elapsed":false,"ext_pan_id":[221,221,221,221,221,221,221,221],"homeassistant_discovery_topic":"homeassistant","homeassistant_legacy_triggers":true,"homeassistant_status_topic":"hass/status","last_seen":"disable","legacy_api":false,"log_directory":directory,"log_file":"log.txt","log_level":"info","log_output":["console","file"],"log_rotation":true,"log_syslog":{},"pan_id":6754,"report":false,"soft_reset_timeout":0,"timestamp_format":"YYYY-MM-DD HH:mm:ss"},"ban":[],"blocklist":[],"device_options":{},"devices":{"0x000b57fffec6a5b2":{"friendly_name":"bulb","retain":true},"0x000b57fffec6a5b3":{"friendly_name":"bulb_color","retain":false},"0x000b57fffec6a5b4":{"friendly_name":"bulb_color_2","retain":false},"0x000b57fffec6a5b7":{"friendly_name":"bulb_2","retain":false},"0x0017880104a44559":{"friendly_name":"J1_cover"},"0x0017880104e43559":{"friendly_name":"U202DST600ZB"},"0x0017880104e44559":{"friendly_name":"3157100_thermostat"},"0x0017880104e45517":{"friendly_name":"remote","retain":true},"0x0017880104e45518":{"friendly_name":"0x0017880104e45518"},"0x0017880104e45520":{"friendly_name":"button","retain":false},"0x0017880104e45521":{"friendly_name":"button_double_key","retain":false},"0x0017880104e45522":{"friendly_name":"weather_sensor","qos":1,"retain":false},"0x0017880104e45523":{"friendly_name":"occupancy_sensor","retain":false},"0x0017880104e45524":{"friendly_name":"power_plug","retain":false},"0x0017880104e45526":{"friendly_name":"GL-S-007ZS"},"0x0017880104e45529":{"friendly_name":"unsupported2","retain":false},"0x0017880104e45530":{"friendly_name":"button_double_key_interviewing","retain":false},"0x0017880104e45540":{"friendly_name":"ikea_onoff"},"0x0017880104e45541":{"friendly_name":"wall_switch","retain":false},"0x0017880104e45542":{"friendly_name":"wall_switch_double","retain":false},"0x0017880104e45543":{"friendly_name":"led_controller_1","retain":false},"0x0017880104e45544":{"friendly_name":"led_controller_2","retain":false},"0x0017880104e45545":{"friendly_name":"dimmer_wall_switch","retain":false},"0x0017880104e45547":{"friendly_name":"curtain","retain":false},"0x0017880104e45548":{"friendly_name":"fan","retain":false},"0x0017880104e45549":{"friendly_name":"siren","retain":false},"0x0017880104e45550":{"friendly_name":"thermostat","retain":false},"0x0017880104e45551":{"friendly_name":"smart vent","retain":false},"0x0017880104e45552":{"friendly_name":"j1","retain":false},"0x0017880104e45553":{"friendly_name":"bulb_enddevice","retain":false},"0x0017880104e45559":{"friendly_name":"cc2530_router","retain":false},"0x0017880104e45560":{"friendly_name":"livolo","retain":false},"0x0017882104a44559":{"friendly_name":"TS0601_thermostat"},"0x90fd9ffffe4b64aa":{"friendly_name":"SP600_OLD"},"0x90fd9ffffe4b64ab":{"friendly_name":"SP600_NEW"},"0x90fd9ffffe4b64ac":{"friendly_name":"MKS-CM-W5"},"0x90fd9ffffe4b64ae":{"friendly_name":"tradfri_remote","retain":false},"0x90fd9ffffe4b64af":{"friendly_name":"roller_shutter"},"0x90fd9ffffe4b64ax":{"friendly_name":"ZNLDP12LM"}},"experimental":{"new_api":true,"output":"json"},"external_converters":[],"groups":{"1":{"friendly_name":"group_1","retain":false},"11":{"devices":["bulb_2"],"friendly_name":"group_with_tradfri","retain":false},"14":{"devices":["power_plug"],"friendly_name":"switch_group","retain":false},"12":{"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","retain":false},"15071":{"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","retain":false},"2":{"friendly_name":"group_2","retain":false}},"homeassistant":false,"map_options":{"graphviz":{"colors":{"fill":{"coordinator":"#e04e5d","enddevice":"#fff8ce","router":"#4ea3e0"},"font":{"coordinator":"#ffffff","enddevice":"#000000","router":"#ffffff"},"line":{"active":"#009900","inactive":"#994444"}}}},"mqtt":{"base_topic":"zigbee2mqtt","include_device_information":false,"server":"mqtt://localhost"},"passlist":[],"permit_join":true,"serial":{"disable_led":false,"port":"/dev/dummy"},"whitelist":[]},"coordinator":{"meta":{"revision":20190425,"version":1},"type":"z-Stack"},"log_level":"info","network":{"channel":15,"extended_pan_id":[0,11,22],"pan_id":5674},"permit_join":false,"version":version.version}),
+            stringify({ "commit": version.commitHash, "config": { "advanced": { "adapter_concurrent": null, "availability_blacklist": [], "availability_blocklist": [], "availability_passlist": [], "availability_timeout": 0, "availability_whitelist": [], "cache_state": true, "cache_state_persistent": true, "cache_state_send_on_startup": true, "channel": 11, "elapsed": false, "ext_pan_id": [221, 221, 221, 221, 221, 221, 221, 221], "homeassistant_discovery_topic": "homeassistant", "homeassistant_legacy_triggers": true, "homeassistant_status_topic": "hass/status", "last_seen": "disable", "legacy_api": false, "log_directory": directory, "log_file": "log.txt", "log_level": "info", "log_output": ["console", "file"], "log_rotation": true, "log_syslog": {}, "pan_id": 6754, "report": false, "soft_reset_timeout": 0, "timestamp_format": "YYYY-MM-DD HH:mm:ss" }, "ban": [], "blocklist": [], "device_options": {}, "devices": { "0x000b57fffec6a5b2": { "friendly_name": "bulb", "retain": true }, "0x000b57fffec6a5b3": { "friendly_name": "bulb_color", "retain": false }, "0x000b57fffec6a5b4": { "friendly_name": "bulb_color_2", "retain": false }, "0x000b57fffec6a5b7": { "friendly_name": "bulb_2", "retain": false }, "0x0017880104a44559": { "friendly_name": "J1_cover" }, "0x0017880104e43559": { "friendly_name": "U202DST600ZB" }, "0x0017880104e44559": { "friendly_name": "3157100_thermostat" }, "0x0017880104e45517": { "friendly_name": "remote", "retain": true }, "0x0017880104e45518": { "friendly_name": "0x0017880104e45518" }, "0x0017880104e45520": { "friendly_name": "button", "retain": false }, "0x0017880104e45521": { "friendly_name": "button_double_key", "retain": false }, "0x0017880104e45522": { "friendly_name": "weather_sensor", "qos": 1, "retain": false }, "0x0017880104e45523": { "friendly_name": "occupancy_sensor", "retain": false }, "0x0017880104e45524": { "friendly_name": "power_plug", "retain": false }, "0x0017880104e45526": { "friendly_name": "GL-S-007ZS" }, "0x0017880104e45529": { "friendly_name": "unsupported2", "retain": false }, "0x0017880104e45530": { "friendly_name": "button_double_key_interviewing", "retain": false }, "0x0017880104e45540": { "friendly_name": "ikea_onoff" }, "0x0017880104e45541": { "friendly_name": "wall_switch", "retain": false }, "0x0017880104e45542": { "friendly_name": "wall_switch_double", "retain": false }, "0x0017880104e45543": { "friendly_name": "led_controller_1", "retain": false }, "0x0017880104e45544": { "friendly_name": "led_controller_2", "retain": false }, "0x0017880104e45545": { "friendly_name": "dimmer_wall_switch", "retain": false }, "0x0017880104e45547": { "friendly_name": "curtain", "retain": false }, "0x0017880104e45548": { "friendly_name": "fan", "retain": false }, "0x0017880104e45549": { "friendly_name": "siren", "retain": false }, "0x0017880104e45550": { "friendly_name": "thermostat", "retain": false }, "0x0017880104e45551": { "friendly_name": "smart vent", "retain": false }, "0x0017880104e45552": { "friendly_name": "j1", "retain": false }, "0x0017880104e45553": { "friendly_name": "bulb_enddevice", "retain": false }, "0x0017880104e45559": { "friendly_name": "cc2530_router", "retain": false }, "0x0017880104e45560": { "friendly_name": "livolo", "retain": false }, "0x0017882104a44559": { "friendly_name": "TS0601_thermostat" }, "0x90fd9ffffe4b64aa": { "friendly_name": "SP600_OLD" }, "0x90fd9ffffe4b64ab": { "friendly_name": "SP600_NEW" }, "0x90fd9ffffe4b64ac": { "friendly_name": "MKS-CM-W5" }, "0x90fd9ffffe4b64ae": { "friendly_name": "tradfri_remote", "retain": false }, "0x90fd9ffffe4b64af": { "friendly_name": "roller_shutter" }, "0x90fd9ffffe4b64ax": { "friendly_name": "ZNLDP12LM" } }, "experimental": { "new_api": true, "output": "json" }, "external_converters": [], "groups": { "1": { "friendly_name": "group_1", "retain": false }, "11": { "devices": ["bulb_2"], "friendly_name": "group_with_tradfri", "retain": false }, "14": { "devices": ["power_plug"], "friendly_name": "switch_group", "retain": false }, "12": { "devices": ["TS0601_thermostat"], "friendly_name": "thermostat_group", "retain": false }, "15071": { "devices": ["bulb_color_2", "bulb_2"], "friendly_name": "group_tradfri_remote", "retain": false }, "2": { "friendly_name": "group_2", "retain": false } }, "homeassistant": false, "map_options": { "graphviz": { "colors": { "fill": { "coordinator": "#e04e5d", "enddevice": "#fff8ce", "router": "#4ea3e0" }, "font": { "coordinator": "#ffffff", "enddevice": "#000000", "router": "#ffffff" }, "line": { "active": "#009900", "inactive": "#994444" } } } }, "mqtt": { "base_topic": "zigbee2mqtt", "include_device_information": false, "server": "mqtt://localhost", "force_disable_retain": false,  }, "passlist": [], "permit_join": true, "serial": { "disable_led": false, "port": "/dev/dummy" }, "whitelist": [] }, "coordinator": { "meta": { "revision": 20190425, "version": 1 }, "type": "z-Stack" }, "log_level": "info", "network": { "channel": 15, "extended_pan_id": [0, 11, 22], "pan_id": 5674 }, 
Line truncated
           { retain: true, qos: 0 },
           expect.any(Function)
         );

test/controller.test.js

@@ -598,4 +598,24 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
     });
 
+    it('Start controller with force_disable_retain', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state" },
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
+    });
+
+    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.mqtt.connect()
+        MQTT.publish.mockClear();
+        await controller.mqtt.publish('fo', 'bar', { retain: true })
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
+    });
 });

test/settings.test.js

@@ -29,7 +29,7 @@ describe('Settings', () => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
     const clearEnvironmentVariables = () => {
-        Object.keys(process.env).forEach((key) => { 
+        Object.keys(process.env).forEach((key) => {
             if(key.indexOf('ZIGBEE2MQTT_CONFIG_') >= 0) {
                 delete process.env[key];
             }
@@ -153,6 +153,7 @@ describe('Settings', () => {
 
         const expected = {
             include_device_information: false,
+            force_disable_retain: false,
             password: "mysecretpassword",
             server: "my.mqtt.server",
             user: "mysecretusername",