1.28.0

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.github/ISSUE_TEMPLATE/new_device_support.yaml

@@ -8,6 +8,7 @@ body:
       value: |
         **IMPORTANT:** Before submitting:
         - Make sure this device is not already supported in the [dev branch](https://www.zigbee2mqtt.io/advanced/more/switch-to-dev-branch.html)
+        - Make sure there is no existing issue or PR for this device already, search for your device here: https://github.com/Koenkk/zigbee2mqtt/issues
         - Follow this [guide](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html)
         - If you are using the Home Assistant addon and are still on 1.18.1, check the first point of [this](https://github.com/Koenkk/zigbee2mqtt/releases/tag/1.19.0)
   - type: input

.github/workflows/ci.yml

@@ -85,24 +85,17 @@ jobs:
     continue-on-error: true
     steps:
       - uses: actions/checkout@v3
-      - name: Cache node-gyp
-        uses: actions/cache@v3        
-        with:
-          key: ${{ matrix.os }}-${{ matrix.node }}-node-gyp
-          path: |
-            /home/runner/.cache/node-gyp
-            C:\Users\runneradmin\AppData\Local\node-gyp
-            /Users/runner/Library/Caches/node-gyp
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node }}
           registry-url: https://registry.npmjs.org/
           cache: 'npm'
       - name: Install dependencies
-        run: npm ci
+        # --ignore-scripts prevents the serialport build which often fails on Windows
+        run: npm ci --ignore-scripts
       - name: Lint
         run: npm run eslint
       - name: Build
         run: npm run build
       - name: Test
-        run: npm run test-with-coverage
+        run: npm run test-with-coverage

.github/workflows/depsreview.yaml

@@ -11,4 +11,4 @@ jobs:
       - name: 'Checkout repository'
         uses: actions/checkout@v3
       - name: 'Dependency review'
-        uses: actions/dependency-review-action@v1
+        uses: actions/dependency-review-action@v2

.github/workflows/stale.yml

@@ -14,7 +14,7 @@ jobs:
       pull-requests: write  # for actions/stale to close stale PRs
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v5
+    - uses: actions/stale@v6
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
@@ -22,4 +22,4 @@ jobs:
         days-before-stale: 30
         days-before-close: 7
         operations-per-run: 500
-        exempt-issue-labels: 'breaking change'
+        exempt-issue-labels: dont-stale

lib/controller.ts

@@ -144,8 +144,7 @@ class Controller {
         try {
             await this.mqtt.connect();
         } catch (error) {
-            logger.error(`MQTT failed to connect: ${error.message}`);
-            logger.error('Exiting...');
+            logger.error(`MQTT failed to connect, exiting...`);
             await this.zigbee.stop();
             await this.exit(1);
         }
@@ -157,7 +156,7 @@ class Controller {
         if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
             for (const entity of [...devices, ...this.zigbee.groups()]) {
                 if (this.state.exists(entity)) {
-                    this.publishEntityState(entity, this.state.get(entity));
+                    this.publishEntityState(entity, this.state.get(entity), 'publishCached');
                 }
             }
         }
@@ -265,10 +264,8 @@ class Controller {
             extension.adjustMessageBeforePublish?.(entity, message);
         }
 
-        // filter mqtt message attributes
-        if (entity.options.filtered_attributes) {
-            entity.options.filtered_attributes.forEach((a) => delete message[a]);
-        }
+        // Filter mqtt message attributes
+        utils.filterProperties(entity.options.filtered_attributes, message);
 
         if (Object.entries(message).length) {
             const output = settings.get().advanced.output;

lib/extension/bind.ts

@@ -13,7 +13,8 @@ const legacyApi = settings.get().advanced.legacy_api;
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
-    'hvacThermostat', 'msTemperatureMeasurement'];
+    'hvacThermostat', 'msIlluminanceMeasurement', 'msTemperatureMeasurement', 'msRelativeHumidity',
+    'msSoilMoisture', 'msCO2'];
 
 // See zigbee-herdsman-converters
 const defaultBindGroup = {type: 'group_number', ID: 901, name: 'default_bind_group'};

lib/extension/bridge.ts

@@ -44,6 +44,7 @@ export default class Bridge extends Extension {
             'backup': this.backup,
             'touchlink/factory_reset': this.touchlinkFactoryReset,
             'touchlink/identify': this.touchlinkIdentify,
+            'install_code/add': this.installCodeAdd,
             'touchlink/scan': this.touchlinkScan,
             'health_check': this.healthCheck,
             'options': this.bridgeOptions,
@@ -248,6 +249,17 @@ export default class Bridge extends Extension {
         return utils.getResponse(message, {zip: base64Zip}, null);
     }
 
+    @bind async installCodeAdd(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        const value = typeof message === 'object' ? message.value : message;
+        await this.zigbee.addInstallCode(value);
+        logger.info('Successfully added new install code');
+        return utils.getResponse(message, {value}, null);
+    }
+
     @bind async permitJoin(message: KeyValue | string): Promise<MQTTResponse> {
         if (typeof message === 'object' && !message.hasOwnProperty('value')) {
             throw new Error('Invalid payload');

lib/extension/groups.ts

@@ -101,7 +101,7 @@ export default class Groups extends Extension {
 
     @bind async onStateChange(data: eventdata.StateChange): Promise<void> {
         const reason = 'groupOptimistic';
-        if (data.reason === reason) {
+        if (data.reason === reason || data.reason === 'publishCached') {
             return;
         }
 
@@ -292,6 +292,7 @@ export default class Groups extends Extension {
             groupKey, deviceKey, skipDisableReporting, resolvedEntityEndpoint,
         } = parsed;
         const message = utils.parseJSON(data.message, data.message);
+        let changedGroups: Group[] = [];
 
         const responseData: KeyValue = {device: deviceKey};
         if (groupKey) {
@@ -320,6 +321,7 @@ export default class Groups extends Extension {
                     logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
                     await resolvedEntityEndpoint.addToGroup(resolvedEntityGroup.zh);
                     settings.addDeviceToGroup(resolvedEntityGroup.ID.toString(), keys);
+                    changedGroups.push(resolvedEntityGroup);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
@@ -333,6 +335,7 @@ export default class Groups extends Extension {
                     logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
                     await resolvedEntityEndpoint.removeFromGroup(resolvedEntityGroup.zh);
                     settings.removeDeviceFromGroup(resolvedEntityGroup.ID.toString(), keys);
+                    changedGroups.push(resolvedEntityGroup);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
@@ -344,6 +347,7 @@ export default class Groups extends Extension {
                     }
                 } else { // remove_all
                     logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    changedGroups = this.zigbee.groups().filter((g) => g.zh.members.includes(resolvedEntityEndpoint));
                     await resolvedEntityEndpoint.removeFromAllGroups();
                     for (const settingsGroup of settings.getGroups()) {
                         settings.removeDeviceFromGroup(settingsGroup.ID.toString(), keys);
@@ -372,8 +376,10 @@ export default class Groups extends Extension {
         if (error) {
             logger.error(error);
         } else {
-            this.eventBus.emitGroupMembersChanged({
-                group: resolvedEntityGroup, action: type, endpoint: resolvedEntityEndpoint, skipDisableReporting});
+            for (const group of changedGroups) {
+                this.eventBus.emitGroupMembersChanged({
+                    group, action: type, endpoint: resolvedEntityEndpoint, skipDisableReporting});
+            }
         }
     }
 }

lib/extension/homeassistant.ts

@@ -282,6 +282,15 @@ export default class HomeAssistant extends Extension {
                 discoveryEntry.discovery_payload.fan_mode_state_topic = true;
             }
 
+            const swingMode = firstExpose.features.find((f) => f.name === 'swing_mode');
+            if (swingMode) {
+                discoveryEntry.discovery_payload.swing_modes = swingMode.values;
+                discoveryEntry.discovery_payload.swing_mode_command_topic = true;
+                discoveryEntry.discovery_payload.swing_mode_state_template =
+                    `{{ value_json.${swingMode.property} }}`;
+                discoveryEntry.discovery_payload.swing_mode_state_topic = true;
+            }
+
             const preset = firstExpose.features.find((f) => f.name === 'preset');
             if (preset) {
                 discoveryEntry.discovery_payload.preset_modes = preset.values;
@@ -302,6 +311,7 @@ export default class HomeAssistant extends Extension {
                         command_topic: true,
                         command_topic_prefix: endpoint,
                         command_topic_postfix: tempCalibration.property,
+                        device_class: 'temperature',
                         entity_category: 'config',
                         icon: 'mdi:math-compass',
                         ...(tempCalibration.unit && {unit_of_measurement: tempCalibration.unit}),
@@ -370,59 +380,56 @@ export default class HomeAssistant extends Extension {
 
             discoveryEntries.push(discoveryEntry);
         } else if (firstExpose.type === 'cover') {
-            const position = exposes.find((expose) => expose.features.find((e) => e.name === 'position'));
-            const tilt = exposes.find((expose) => expose.features.find((e) => e.name === 'tilt'));
+            const state = exposes.find((expose) => expose.features.find((e) => e.name === 'state'))
+                ?.features.find((f) => f.name === 'state');
+            const position = exposes.find((expose) => expose.features.find((e) => e.name === 'position'))
+                ?.features.find((f) => f.name === 'position');
+            const tilt = exposes.find((expose) => expose.features.find((e) => e.name === 'tilt'))
+                ?.features.find((f) => f.name === 'tilt');
             const motorState = definitionExposes?.find((e) => e.type === 'enum' && e.name === 'motor_state' &&
                 e.access === ACCESS_STATE);
+            const running = definitionExposes?.find((e) => e.type === 'binary' && e.name === 'running');
 
             const discoveryEntry: DiscoveryEntry = {
                 type: 'cover',
-                mockProperties: [],
+                mockProperties: [{property: state.property, value: null}],
                 object_id: endpoint ? `cover_${endpoint}` : 'cover',
                 discovery_payload: {
                     command_topic_prefix: endpoint,
+                    command_topic: true,
+                    state_topic: true,
+                    state_topic_postfix: endpoint,
                 },
             };
 
-            // For covers only supporting tilt don't discover the command/state_topic, otherwise
-            // HA does not correctly reflect the state
-            // - https://github.com/home-assistant/core/issues/51793
-            // - https://github.com/Koenkk/zigbee-herdsman-converters/pull/2663
-            if (!tilt || (tilt && position)) {
-                discoveryEntry.discovery_payload.command_topic = true;
-                discoveryEntry.discovery_payload.state_topic = !position;
-                discoveryEntry.discovery_payload.command_topic_prefix = endpoint;
+            // For curtains that have `motor_state` lookup a possible state names and make this
+            // available for discovery. If the curtains only support the `running` value,
+            // then we use it anyway. The movement direction is calculated (assumed) in this case.
+            if (motorState) {
+                const openingLookup = ['opening', 'open', 'forward', 'up', 'rising'];
+                const closingLookup = ['closing', 'close', 'backward', 'back', 'reverse', 'down', 'declining'];
+                const stoppedLookup = ['stopped', 'stop', 'pause', 'paused'];
 
-                // For curtains that have `motor_state` lookup a possible state names and make this
-                // available for discovery. If the curtains only support the `running` value,
-                // then we use it anyway. The movement direction is calculated (assumed) in this case.
-                if (motorState) {
-                    const openingLookup = ['opening', 'open', 'forward', 'up', 'rising'];
-                    const closingLookup = ['closing', 'close', 'backward', 'back', 'reverse', 'down', 'declining'];
-                    const stoppedLookup = ['stopped', 'stop', 'pause', 'paused'];
+                const openingState = motorState.values.find((s) => openingLookup.includes(s.toLowerCase()));
+                const closingState = motorState.values.find((s) => closingLookup.includes(s.toLowerCase()));
+                const stoppedState = motorState.values.find((s) => stoppedLookup.includes(s.toLowerCase()));
 
-                    const openingState = motorState.values.find((s) => openingLookup.includes(s.toLowerCase()));
-                    const closingState = motorState.values.find((s) => closingLookup.includes(s.toLowerCase()));
-                    const stoppedState = motorState.values.find((s) => stoppedLookup.includes(s.toLowerCase()));
-
-                    if (openingState && closingState && stoppedState) {
-                        discoveryEntry.discovery_payload.state_topic = true;
-                        discoveryEntry.discovery_payload.state_opening = openingState;
-                        discoveryEntry.discovery_payload.state_closing = closingState;
-                        discoveryEntry.discovery_payload.state_stopped = stoppedState;
-                        discoveryEntry.discovery_payload.value_template = `{% if not value_json.motor_state %} ` +
-                            `${stoppedState} {% else %} {{ value_json.motor_state }} {% endif %}`;
-                    }
-                } else {
-                    const running = definitionExposes?.find((e) => e.type === 'binary' && e.name === 'running');
-
-                    if (running) {
-                        discoveryEntry.discovery_payload.state_topic = true;
-                        discoveryEntry.discovery_payload.value_template = `{% if not value_json.running %} ` +
-                            `stopped {% else %} {% if value_json.position > 0 %} closing {% else %} ` +
-                            `opening {% endif %} {% endif %}`;
-                    }
+                if (openingState && closingState && stoppedState) {
+                    discoveryEntry.discovery_payload.state_opening = openingState;
+                    discoveryEntry.discovery_payload.state_closing = closingState;
+                    discoveryEntry.discovery_payload.state_stopped = stoppedState;
+                    discoveryEntry.discovery_payload.value_template = `{% if not value_json.${motorState.property} %}` +
+                        ` ${stoppedState} {% else %} {{ value_json.${motorState.property} }} {% endif %}`;
                 }
+            } else if (running) {
+                discoveryEntry.discovery_payload.value_template = `{% if not value_json.${running.property} %} ` +
+                    `stopped {% else %} {% if value_json.${position.property} > 0 %} closing {% else %} ` +
+                    `opening {% endif %} {% endif %}`;
+            } else {
+                discoveryEntry.discovery_payload.value_template =
+                    `{{ value_json.${featurePropertyWithoutEndpoint(state)} }}`,
+                discoveryEntry.discovery_payload.state_open = 'OPEN';
+                discoveryEntry.discovery_payload.state_closed = 'CLOSE';
             }
 
             if (!position && !tilt) {
@@ -430,21 +437,19 @@ export default class HomeAssistant extends Extension {
             }
 
             if (position) {
-                const p = position.features.find((f) => f.name === 'position');
                 discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
-                    position_template: `{{ value_json.${getProperty(p)} }}`,
-                    set_position_template: `{ "${getProperty(p)}": {{ position }} }`,
+                    position_template: `{{ value_json.${featurePropertyWithoutEndpoint(position)} }}`,
+                    set_position_template: `{ "${getProperty(position)}": {{ position }} }`,
                     set_position_topic: true,
                     position_topic: true,
                 };
             }
 
             if (tilt) {
-                const t = tilt.features.find((f) => f.name === 'tilt');
                 discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
                     tilt_command_topic: true,
                     tilt_status_topic: true,
-                    tilt_status_template: `{{ value_json.${getProperty(t)} }}`,
+                    tilt_status_template: `{{ value_json.${featurePropertyWithoutEndpoint(tilt)} }}`,
                 };
             }
 
@@ -528,6 +533,7 @@ export default class HomeAssistant extends Extension {
                 garage_door_contact: {device_class: 'garage_door', payload_on: false, payload_off: true},
                 eco_mode: {entity_category: 'config', icon: 'mdi:leaf'},
                 expose_pin: {entity_category: 'config', icon: 'mdi:pin'},
+                flip_indicator_light: {entity_category: 'config', icon: 'mdi:arrow-left-right'},
                 gas: {device_class: 'gas'},
                 invert_cover: {entity_category: 'config', icon: 'mdi:arrow-left-right'},
                 led_disabled_night: {entity_category: 'config', icon: 'mdi:led-off'},
@@ -727,7 +733,7 @@ export default class HomeAssistant extends Extension {
                 discovery_payload: {
                     value_template: `{{ value_json.${firstExpose.property} }}`,
                     enabled_by_default: !allowsSet,
-                    ...(firstExpose.unit && {unit_of_measurement: firstExpose.unit}),
+                    unit_of_measurement: firstExpose.unit || '',
                     ...lookup[firstExpose.name],
                     ...extraAttrs,
                 },
@@ -754,6 +760,12 @@ export default class HomeAssistant extends Extension {
                     },
                 };
 
+                if (lookup[firstExpose.name]?.device_class === 'temperature') {
+                    discoveryEntry.discovery_payload.device_class == lookup[firstExpose.name]?.device_class;
+                } else {
+                    delete discoveryEntry.discovery_payload.device_class;
+                }
+
                 if (firstExpose.value_min != null) discoveryEntry.discovery_payload.min = firstExpose.value_min;
                 if (firstExpose.value_max != null) discoveryEntry.discovery_payload.max = firstExpose.value_max;
 
@@ -795,39 +807,42 @@ export default class HomeAssistant extends Extension {
                 week: {entity_category: 'config', icon: 'mdi:calendar-clock'},
             };
 
+            const valueTemplate = firstExpose.access & ACCESS_STATE ?
+                `{{ value_json.${firstExpose.property} }}` : undefined;
+
             if (firstExpose.access & ACCESS_STATE) {
                 discoveryEntries.push({
                     type: 'sensor',
                     object_id: firstExpose.property,
                     mockProperties: [{property: firstExpose.property, value: null}],
                     discovery_payload: {
-                        value_template: `{{ value_json.${firstExpose.property} }}`,
+                        value_template: valueTemplate,
                         enabled_by_default: !(firstExpose.access & ACCESS_SET),
                         ...lookup[firstExpose.name],
                     },
                 });
+            }
 
-                /**
-                 * If enum attribute has SET access then expose as SELECT entity too.
-                 * Note: currently both sensor and select are discovered, this is to avoid
-                 * breaking changes for sensors already existing in HA (legacy).
-                 */
-                if ((firstExpose.access & ACCESS_SET)) {
-                    discoveryEntries.push({
-                        type: 'select',
-                        object_id: firstExpose.property,
-                        mockProperties: [{property: firstExpose.property, value: null}],
-                        discovery_payload: {
-                            value_template: `{{ value_json.${firstExpose.property} }}`,
-                            state_topic: true,
-                            command_topic_prefix: endpoint,
-                            command_topic: true,
-                            command_topic_postfix: firstExpose.property,
-                            options: firstExpose.values.map((v) => v.toString()),
-                            ...lookup[firstExpose.name],
-                        },
-                    });
-                }
+            /**
+             * If enum attribute has SET access then expose as SELECT entity too.
+             * Note: currently both sensor and select are discovered, this is to avoid
+             * breaking changes for sensors already existing in HA (legacy).
+             */
+            if ((firstExpose.access & ACCESS_SET)) {
+                discoveryEntries.push({
+                    type: 'select',
+                    object_id: firstExpose.property,
+                    mockProperties: [], // Already mocked above in case access STATE is supported
+                    discovery_payload: {
+                        value_template: valueTemplate,
+                        state_topic: !!(firstExpose.access & ACCESS_STATE),
+                        command_topic_prefix: endpoint,
+                        command_topic: true,
+                        command_topic_postfix: firstExpose.property,
+                        options: firstExpose.values.map((v) => v.toString()),
+                        ...lookup[firstExpose.name],
+                    },
+                });
             }
         } else if (firstExpose.type === 'text' || firstExpose.type === 'composite') {
             if (firstExpose.access & ACCESS_STATE) {
@@ -878,7 +893,11 @@ export default class HomeAssistant extends Extension {
         const entity = this.zigbee.resolveEntity(data.entity.name);
         if (entity.isDevice() && this.discovered[entity.ieeeAddr]) {
             for (const objectID of this.discovered[entity.ieeeAddr].objectIDs) {
-                const match = /light_(.*)/.exec(objectID);
+                const lightMatch = /light_(.*)/.exec(objectID);
+                const coverMatch = /cover_(.*)/.exec(objectID);
+
+                const match = lightMatch || coverMatch;
+
                 if (match) {
                     const endpoint = match[1];
                     const endpointRegExp = new RegExp(`(.*)_${endpoint}`);
@@ -1218,6 +1237,14 @@ export default class HomeAssistant extends Extension {
                 payload.fan_mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/fan_mode`;
             }
 
+            if (payload.swing_mode_state_topic) {
+                payload.swing_mode_state_topic = stateTopic;
+            }
+
+            if (payload.swing_mode_command_topic) {
+                payload.swing_mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/swing_mode`;
+            }
+
             if (payload.percentage_state_topic) {
                 payload.percentage_state_topic = stateTopic;
             }
@@ -1336,7 +1363,7 @@ export default class HomeAssistant extends Extension {
                 // Publish all device states.
                 for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
                     if (this.state.exists(entity)) {
-                        this.publishEntityState(entity, this.state.get(entity));
+                        this.publishEntityState(entity, this.state.get(entity), 'publishCached');
                     }
                 }
 

lib/extension/publish.ts

@@ -256,7 +256,8 @@ export default class Publish extends Extension {
                         }
 
                         // filter out attribute listed in filtered_optimistic
-                        entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
+                        utils.filterProperties(entitySettings.filtered_optimistic, msg);
+
                         addToToPublish(re, msg);
                     }
 

lib/extension/receive.ts

@@ -145,9 +145,9 @@ export default class Receive extends Extension {
                 if (converted) {
                     payload = {...payload, ...converted};
                 }
-            } catch (error) {
-                // istanbul ignore next
+            } catch (error) /* istanbul ignore next */ {
                 logger.error(`Exception while calling fromZigbee converter: ${error.message}}`);
+                logger.debug(error.stack);
             }
         }
 

lib/mqtt.ts

@@ -10,6 +10,10 @@ export default class MQTT {
     private connectionTimer: NodeJS.Timeout;
     private client: mqtt.MqttClient;
     private eventBus: EventBus;
+    private initialConnect = true;
+    private republishRetainedTimer: NodeJS.Timer;
+    private retainedMessages: {[s: string]: {payload: string, options: MQTTOptions,
+        skipLog: boolean, skipReceive: boolean, topic: string, base: string}} = {};
 
     constructor(eventBus: EventBus) {
         this.eventBus = eventBus;
@@ -79,7 +83,10 @@ export default class MQTT {
                 resolve();
             });
 
-            this.client.on('error', (err) => reject(err));
+            this.client.on('error', (err) => {
+                logger.error(`MQTT error: ${err.message}`);
+                reject(err);
+            });
             this.client.on('message', this.onMessage);
         });
     }
@@ -94,8 +101,19 @@ export default class MQTT {
         }, utils.seconds(10));
 
         logger.info('Connected to MQTT server');
-        this.subscribe(`${settings.get().mqtt.base_topic}/#`);
         await this.publishStateOnline();
+
+        if (!this.initialConnect) {
+            this.republishRetainedTimer = setTimeout(() => {
+                // Republish retained messages in case MQTT broker does not persist them.
+                // https://github.com/Koenkk/zigbee2mqtt/issues/9629
+                Object.values(this.retainedMessages).forEach((e) =>
+                    this.publish(e.topic, e.payload, e.options, e.base, e.skipLog, e.skipReceive));
+            }, 2000);
+        }
+
+        this.initialConnect = false;
+        this.subscribe(`${settings.get().mqtt.base_topic}/#`);
     }
 
     async publishStateOnline(): Promise<void> {
@@ -121,6 +139,11 @@ export default class MQTT {
             logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
             this.eventBus.emitMQTTMessage({topic, message: message + ''});
         }
+
+        if (this.republishRetainedTimer && topic == `${settings.get().mqtt.base_topic}/bridge/state`) {
+            clearTimeout(this.republishRetainedTimer);
+            this.republishRetainedTimer = null;
+        }
     }
 
     isConnected(): boolean {
@@ -137,6 +160,15 @@ export default class MQTT {
             this.publishedTopics.add(topic);
         }
 
+        if (options.retain) {
+            if (payload) {
+                this.retainedMessages[topic] =
+                    {payload, options, skipReceive, skipLog, topic: topic.substring(base.length + 1), base};
+            } else {
+                delete this.retainedMessages[topic];
+            }
+        }
+
         this.eventBus.emitMQTTMessagePublished({topic, payload, options: {...defaultOptions, ...options}});
 
         if (!this.isConnected()) {

lib/state.ts

@@ -1,15 +1,16 @@
 import logger from './util/logger';
 import data from './util/data';
 import * as settings from './util/settings';
+import utils from './util/utils';
 import fs from 'fs';
 import objectAssignDeep from 'object-assign-deep';
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
-    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
-    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];
 
 class State {
@@ -74,17 +75,14 @@ class State {
     set(entity: Group | Device, update: KeyValue, reason: string=null): KeyValue {
         const fromState = this.state[entity.ID] || {};
         const toState = objectAssignDeep({}, fromState, update);
-        const result = {...toState};
+        const newCache = {...toState};
+        const entityDontCacheProperties = entity.options.filtered_cache || [];
 
-        for (const property of Object.keys(toState)) {
-            if (dontCacheProperties.find((p) => property.match(p))) {
-                delete toState[property];
-            }
-        }
+        utils.filterProperties(dontCacheProperties.concat(entityDontCacheProperties), newCache);
 
-        this.state[entity.ID] = toState;
+        this.state[entity.ID] = newCache;
         this.eventBus.emitStateChange({entity, from: fromState, to: toState, reason, update});
-        return result;
+        return toState;
     }
 
     remove(ID: string | number): void {

lib/types/types.d.ts

@@ -42,7 +42,7 @@ declare global {
     // Types
     interface MQTTResponse {data: KeyValue, status: 'error' | 'ok', error?: string, transaction?: string}
     interface MQTTOptions {qos?: mqtt.QoS, retain?: boolean, properties?: {messageExpiryInterval: number}}
-    type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged';
+    type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged' | 'publishCached';
     type PublishEntityState = (entity: Device | Group, payload: KeyValue,
         stateChangeReason?: StateChangeReason) => Promise<void>;
     type RecursivePartial<T> = {[P in keyof T]?: RecursivePartial<T[P]>;};
@@ -280,11 +280,12 @@ declare global {
         retrieve_state?: boolean,
         debounce?: number,
         debounce_ignore?: string[],
+        filtered_attributes?: string[],
+        filtered_cache?: string[],
         filtered_optimistic?: string[],
         icon?: string,
         homeassistant?: KeyValue,
         legacy?: boolean,
-        filtered_attributes?: string[],
         friendly_name: string,
         description?: string,
         qos?: 0 | 1 | 2,
@@ -295,10 +296,11 @@ declare global {
         ID?: number,
         optimistic?: boolean,
         off_state?: 'all_members_off' | 'last_member_state'
+        filtered_attributes?: string[],
+        filtered_cache?: string[],
         filtered_optimistic?: string[],
         retrieve_state?: boolean,
         homeassistant?: KeyValue,
-        filtered_attributes?: string[],
         friendly_name: string,
         description?: string,
         qos?: 0 | 1 | 2,

lib/util/settings.schema.json

@@ -812,23 +812,32 @@
           "description": "Publish optimistic state after set",
           "default": true
         },
-        "filtered_optimistic": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "examples": ["color_mode", "color_temp", "color"],
-          "title": "Filtered optimistic attributes",
-          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
-        },
         "filtered_attributes": {
           "type": "array",
           "items": {
             "type": "string"
           },
-          "examples": ["temperature", "battery", "action"],
+          "examples": ["^temperature$", "^battery$", "^action$"],
           "title": "Filtered publish attributes",
-          "description": "Filter attributes from publish payload."
+          "description": "Filter attributes with regex from published payload."
+        },
+        "filtered_cache": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["^input_actions$"],
+          "title": "Filtered attributes from cache",
+          "description": "Filter attributes with regex from being added to the cache, this prevents the attribute from being in the published payload when the value didn't change."
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["^color_(mode|temp)$", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes with regex from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
         },
         "icon": {
           "type": "string",

lib/util/settings.ts

@@ -309,6 +309,7 @@ export function validate(): string[] {
 
 function read(): Settings {
     const s = yaml.read(file) as Settings;
+    applyEnvironmentVariables(s);
 
     // Read !secret MQTT username and password if set
     // eslint-disable-next-line
@@ -381,7 +382,11 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
                         }, settings);
 
                         if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
-                            setting[key] = JSON.parse(process.env[envVariableName]);
+                            try {
+                                setting[key] = JSON.parse(process.env[envVariableName]);
+                            } catch (error) {
+                                setting[key] = process.env[envVariableName];
+                            }
                         } else if (type.indexOf('number') >= 0) {
                             /* eslint-disable-line */ // @ts-ignore
                             setting[key] = process.env[envVariableName] * 1;
@@ -412,7 +417,6 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
 function getInternalSettings(): Partial<Settings> {
     if (!_settings) {
         _settings = read();
-        applyEnvironmentVariables(_settings);
     }
 
     return _settings;

lib/util/utils.ts

@@ -341,6 +341,15 @@ function publishLastSeen(data: eventdata.LastSeenChanged, settings: Settings, al
     }
 }
 
+function filterProperties(filter: string[], data: KeyValue): void {
+    if (filter) {
+        for (const property of Object.keys(data)) {
+            if (filter.find((p) => property.match(`^${p}$`))) {
+                delete data[property];
+            }
+        }
+    }
+}
 
 export default {
     endpointNames, capitalize, getZigbee2MQTTVersion, getDependencyVersion, formatDate, objectHasProperties,
@@ -348,5 +357,5 @@ export default {
     getExternalConvertersDefinitions, removeNullPropertiesFromObject, toNetworkAddressHex, toSnakeCase,
     parseEntityID, isEndpoint, isZHGroup, hours, minutes, seconds, validateFriendlyName, sleep,
     sanitizeImageParameter, isAvailabilityEnabledForEntity, publishLastSeen, availabilityPayload,
-    getAllFiles,
+    getAllFiles, filterProperties,
 };

lib/zigbee.ts

@@ -310,6 +310,10 @@ export default class Zigbee {
         return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
     }
 
+    async addInstallCode(installCode: string): Promise<void> {
+        await this.herdsman.addInstallCode(installCode);
+    }
+
     async touchlinkIdentify(ieeeAddr: string, channel: number): Promise<void> {
         await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.26.0",
+  "version": "1.28.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -38,54 +38,54 @@
     "ajv": "^8.11.0",
     "bind-decorator": "^1.0.11",
     "connect-gzip-static": "2.1.1",
-    "core-js": "^3.23.3",
+    "core-js": "^3.25.2",
     "debounce": "^1.2.1",
     "deep-object-diff": "^1.1.7",
     "fast-deep-equal": "^3.1.3",
     "finalhandler": "^1.2.0",
     "git-last-commit": "^1.0.1",
-    "humanize-duration": "^3.27.2",
+    "humanize-duration": "^3.27.3",
     "js-yaml": "^4.1.0",
     "json-stable-stringify-without-jsonify": "^1.0.1",
-    "jszip": "^3.10.0",
+    "jszip": "^3.10.1",
     "mkdir-recursive": "^0.4.0",
-    "moment": "^2.29.3",
+    "moment": "^2.29.4",
     "mqtt": "4.3.7",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
     "semver": "^7.3.7",
     "source-map-support": "^0.5.21",
     "uri-js": "^4.4.1",
-    "winston": "^3.8.0",
-    "winston-syslog": "^2.5.0",
+    "winston": "^3.8.2",
+    "winston-syslog": "^2.6.0",
     "winston-transport": "^4.5.0",
-    "ws": "^8.8.0",
-    "zigbee-herdsman": "0.14.40",
-    "zigbee-herdsman-converters": "14.0.559",
-    "zigbee2mqtt-frontend": "0.6.103"
+    "ws": "^8.9.0",
+    "zigbee-herdsman": "0.14.62",
+    "zigbee-herdsman-converters": "14.0.635",
+    "zigbee2mqtt-frontend": "0.6.112"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.5",
-    "@babel/plugin-proposal-decorators": "^7.18.2",
-    "@babel/preset-env": "^7.18.2",
-    "@babel/preset-typescript": "^7.17.12",
+    "@babel/core": "^7.19.1",
+    "@babel/plugin-proposal-decorators": "^7.19.1",
+    "@babel/preset-env": "^7.19.1",
+    "@babel/preset-typescript": "^7.18.6",
     "@types/debounce": "^1.2.1",
-    "@types/finalhandler": "^1.1.1",
+    "@types/finalhandler": "^1.2.0",
     "@types/humanize-duration": "^3.27.1",
-    "@types/jest": "^28.1.3",
+    "@types/jest": "^29.0.3",
     "@types/js-yaml": "^4.0.5",
     "@types/object-assign-deep": "^0.4.0",
     "@types/rimraf": "^3.0.2",
     "@types/ws": "^8.5.3",
-    "@typescript-eslint/eslint-plugin": "^5.29.0",
-    "@typescript-eslint/parser": "^5.29.0",
-    "babel-jest": "^28.1.1",
-    "eslint": "^8.18.0",
+    "@typescript-eslint/eslint-plugin": "^5.38.0",
+    "@typescript-eslint/parser": "^5.38.0",
+    "babel-jest": "^29.0.3",
+    "eslint": "^8.24.0",
     "eslint-config-google": "^0.14.0",
-    "eslint-plugin-jest": "^26.5.3",
-    "jest": "^28.1.1",
+    "eslint-plugin-jest": "^27.0.4",
+    "jest": "^29.0.3",
     "tmp": "^0.2.1",
-    "typescript": "^4.7.4"
+    "typescript": "^4.8.3"
   },
   "jest": {
     "coverageThreshold": {

test/controller.test.js

@@ -202,7 +202,8 @@ describe('Controller', () => {
         });
         await controller.start();
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith('MQTT failed to connect: addr not found');
+        expect(logger.error).toHaveBeenCalledWith('MQTT error: addr not found');
+        expect(logger.error).toHaveBeenCalledWith('MQTT failed to connect, exiting...');
         expect(mockExit).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledWith(1, false);
     });
@@ -490,6 +491,46 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
+    it('Publish entity state attribute_json output filtered cache', async () => {
+        await controller.start();
+        settings.set(['advanced', 'output'], 'attribute_and_json');
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_cache'], ['linkquality']);
+        MQTT.publish.mockClear();
+
+        const device = controller.zigbee.resolveEntity('bulb');
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
+
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
+        await flushPromises();
+
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output filtered cache (device_options)', async () => {
+        await controller.start();
+        settings.set(['advanced', 'output'], 'attribute_and_json');
+        settings.set(['device_options', 'filtered_cache'], ['linkquality']);
+        MQTT.publish.mockClear();
+
+        const device = controller.zigbee.resolveEntity('bulb');
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
+
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
+        await flushPromises();
+
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
     it('Publish entity state with device information', async () => {
         await controller.start();
         settings.set(['mqtt', 'include_device_information'], true);
@@ -605,6 +646,27 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should republish retained messages on MQTT reconnect', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        MQTT.events['connect']();
+        await flushPromises();
+        jest.runOnlyPendingTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(13);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Should not republish retained messages on MQTT reconnect when retained message are sent', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        MQTT.events['connect']();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/state', 'online');
+        jest.runOnlyPendingTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/state', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
     it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
         settings.set(['mqtt', 'force_disable_retain'], true);
         await controller.mqtt.connect()

test/homeassistant.test.js

@@ -811,6 +811,10 @@ describe('HomeAssistant extension', () => {
             set_position_topic: 'zigbee2mqtt/smart vent/set',
             set_position_template: '{ "position": {{ position }} }',
             position_template: '{{ value_json.position }}',
+            state_topic: 'zigbee2mqtt/smart vent',
+            value_template: `{{ value_json.state }}`,
+            state_open: 'OPEN',
+            state_closed: 'CLOSE',
             json_attributes_topic: 'zigbee2mqtt/smart vent',
             name: 'smart vent',
             unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
@@ -831,6 +835,39 @@ describe('HomeAssistant extension', () => {
             { retain: true, qos: 0 },
             expect.any(Function),
         );
+
+        payload = {
+            "availability": [{ "topic": "zigbee2mqtt/bridge/state" }],
+            "command_topic": "zigbee2mqtt/zigfred_plus/l6/set",
+            "device": {
+                "identifiers": ["zigbee2mqtt_0xf4ce368a38be56a1"],
+                "manufacturer": "Siglis",
+                "model": "zigfred plus smart in-wall switch (ZFP-1A-CH)",
+                "name": "zigfred_plus",
+                "sw_version": null
+            },
+            "json_attributes_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "name": "zigfred_plus_l6",
+            "position_template": "{{ value_json.position }}",
+            "position_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "set_position_template": "{ \"position_l6\": {{ position }} }",
+            "set_position_topic": "zigbee2mqtt/zigfred_plus/l6/set",
+            "state_closed": "CLOSE",
+            "state_open": "OPEN",
+            "state_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "tilt_command_topic": "zigbee2mqtt/zigfred_plus/l6/set/tilt",
+            "tilt_status_template": "{{ value_json.tilt }}",
+            "tilt_status_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "unique_id": "0xf4ce368a38be56a1_cover_l6_zigbee2mqtt",
+            "value_template": "{{ value_json.state }}"
+        }
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/cover/0xf4ce368a38be56a1/cover_l6/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
     });
 
     it('Should discover devices with custom homeassistant_discovery_topic', async () => {

test/publish.test.js

@@ -507,6 +507,18 @@ describe('Publish', () => {
         expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
     });
 
+    it('Should handle get with multiple cover endpoints', async () => {
+        const device = zigbeeHerdsman.devices.zigfred_plus;
+        const endpoint11 = device.getEndpoint(11);
+        const endpoint12 = device.getEndpoint(12);
+        await MQTT.events.message('zigbee2mqtt/zigfred_plus/get', stringify({state_l6: '', state_l7: ''}));
+        await flushPromises();
+        expect(endpoint11.read).toHaveBeenCalledTimes(1);
+        expect(endpoint11.read).toHaveBeenCalledWith('closuresWindowCovering', ['currentPositionLiftPercentage']);
+        expect(endpoint12.read).toHaveBeenCalledTimes(1);
+        expect(endpoint12.read).toHaveBeenCalledWith('closuresWindowCovering', ['currentPositionLiftPercentage']);
+    });
+
     it('Should log error when device has no such endpoint (via topic)', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint2 = device.getEndpoint(2);

test/receive.test.js

@@ -84,7 +84,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20, state: 'OPEN'}), {retain: false, qos: 0}, expect.any(Function));
 
         // Inverted
         MQTT.publish.mockClear();
@@ -92,7 +92,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80, state: 'OPEN'}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should allow to disable the legacy integration', async () => {
@@ -242,15 +242,15 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
-    it('Should handle a zigbee message with voltage 3010', async () => {
+    it('Should handle a zigbee message with voltage 2990', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 3010}}
+        const data = {'65281': {'1': 2990}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 93, voltage: 2990});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 

test/settings.test.js

@@ -67,20 +67,42 @@ describe('Settings', () => {
     it('Should apply environment variables', () => {
         process.env['ZIGBEE2MQTT_CONFIG_SERIAL_DISABLE_LED'] = 'true';
         process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_SOFT_RESET_TIMEOUT'] = 1;
-        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_OUTPUT'] = 'csvtest';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_OUTPUT'] = 'attribute_and_json';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_LOG_OUTPUT'] = '["console"]';
         process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
         process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
+        process.env['ZIGBEE2MQTT_CONFIG_MQTT_SERVER'] = 'testserver';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_NETWORK_KEY'] = 'GENERATE';
+        process.env['ZIGBEE2MQTT_CONFIG_DEVICES'] = 'devices.yaml';
+
+        const contentDevices = {
+            '0x00158d00018255df': {
+                friendly_name: '0x00158d00018255df',
+                retain: false,
+            },
+        };
 
         write(configurationFile, {});
+        write(devicesFile, contentDevices);
+        expect(settings.validate()).toStrictEqual([]);
+
         const s = settings.get();
         const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
-        expected.devices = {};
+        expected.devices = {
+            '0x00158d00018255df': {
+                friendly_name: '0x00158d00018255df',
+                retain: false,
+            },
+        };
         expected.groups = {};
         expected.serial.disable_led = true;
         expected.advanced.soft_reset_timeout = 1;
-        expected.advanced.output = 'csvtest';
+        expected.advanced.log_output = ["console"];
+        expected.advanced.output = 'attribute_and_json';
         expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
         expected.mqtt.base_topic = 'testtopic';
+        expected.mqtt.server = 'testserver';
+        expected.advanced.network_key = 'GENERATE';
 
         expect(s).toStrictEqual(expected);
     });

test/stub/data.js

@@ -153,6 +153,23 @@ function writeDefaultConfiguration() {
             '0x0017880104e43559': {
                 friendly_name: 'U202DST600ZB'
             },
+            '0xf4ce368a38be56a1': {
+                retain: false,
+                friendly_name: 'zigfred_plus',
+                front_surface_enabled: 'true',
+                dimmer_1_enabled: 'true',
+                dimmer_1_dimming_enabled: 'true',
+                dimmer_2_enabled: 'true',
+                dimmer_2_dimming_enabled: 'true',
+                dimmer_3_enabled: 'true',
+                dimmer_3_dimming_enabled: 'true',
+                dimmer_4_enabled: 'true',
+                dimmer_4_dimming_enabled: 'true',
+                cover_1_enabled: 'true',
+                cover_1_tilt_enabled: 'true',
+                cover_2_enabled: 'true',
+                cover_2_tilt_enabled: 'true',
+            },
             '0x0017880104e44559': {
                 friendly_name: '3157100_thermostat',
             },

test/stub/zigbeeHerdsman.js

@@ -27,7 +27,11 @@ const clusters = {
     'lightingColorCtrl': 768,
     'closuresWindowCovering': 258,
     'hvacThermostat': 513,
+    'msIlluminanceMeasurement': 1024,
     'msTemperatureMeasurement': 1026,
+    'msRelativeHumidity': 1029,
+    'msSoilMoisture': 1032,
+    'msCO2': 1037
 }
 
 class Endpoint {
@@ -133,6 +137,7 @@ const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,41
 const ZNCZ02LM = new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug");
 const GLEDOPTO_2ID = new Device('Router', '0x0017880104e45724', 6540,4151, [new Endpoint(11, [0,3,4,5,6,8,768], [], '0x0017880104e45724', [], {}, [], 49246, 528), new Endpoint(12, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 260, 258), new Endpoint(13, [4096], [4096], '0x0017880104e45724', [], {}, [], 49246, 57694), new Endpoint(15, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 49246, 256)], true, "Mains (single phase)", 'GL-C-007', false, 'GLEDOPTO');
 const QBKG03LM = new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2');
+const zigfred_plus = new Device('Router', '0xf4ce368a38be56a1', 6589, 0x129C, [new Endpoint(5, [0, 3, 4, 5, 6, 8, 0x0300, 0xFC42], [0xFC42], '0xf4ce368a38be56a1'), new Endpoint(7, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(8, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(9, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(10, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(11, [0, 3, 4, 5, 0x0102], [], '0xf4ce368a38be56a1'), new Endpoint(12, [0, 3, 4, 5, 0x0102], [], '0xf4ce368a38be56a1')], true, "Mains (single phase)", 'zigfred plus', false, 'Siglis');
 
 const groups = {
     'group_1': new Group(1, []),
@@ -188,6 +193,7 @@ const devices = {
     'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
     'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
     'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
+    'zigfred_plus': zigfred_plus,
     '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100', false, 'Centralite'),
     'J1': new Device('Router', '0x0017880104a44559', 6543,4151, [new Endpoint(1, [], [], '0x0017880104a44559')], true, "Mains (single phase)", 'J1 (5502)'),
     'TS0601_thermostat': TS0601_thermostat,
@@ -197,7 +203,7 @@ const devices = {
     'temperature_sensor': new Device('EndDevice', '0x0017880104e45561', 6544,4151, [new Endpoint(1, [0,3,4,1026], [])], true, "Battery", "temperature.sensor"),
     'heating_actuator': new Device('Router', '0x0017880104e45562', 6545,4151, [new Endpoint(1, [0,3,4,513], [1026])], true, "Mains (single phase)", "heating.actuator"),
     'bj_scene_switch': new Device('EndDevice', '0xd85def11a1002caa', 50117, 4398, [new Endpoint(10, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa', [{target: bulb_color_2.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 6, name: 'genOnOff'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 768, name: 'lightingColorCtrl'}},]), new Endpoint(11, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa')], true, 'Battery', 'RB01', false, 'Busch-Jaeger', '20161222', '1.2.0'),
-
+    'GW003-AS-IN-TE-FC': new Device('Router', '0x0017548104a44669', 6545,4699, [new Endpoint(1, [3], [0,3,513,514], '0x0017548104a44669')], true, "Mains (single phase)", 'Adapter Zigbee FUJITSU'),
 }
 
 const mock = {
@@ -210,6 +216,7 @@ const mock = {
     backup: jest.fn(),
     isStopping: jest.fn(),
     permitJoin: jest.fn(),
+    addInstallCode: jest.fn(),
     getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
     getNetworkParameters: jest.fn().mockReturnValue({panID: 0x162a, extendedPanID: [0, 11, 22], channel: 15}),
     on: (type, handler) => {