1.25.0

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.github/ISSUE_TEMPLATE/new_device_support.yaml

@@ -50,5 +50,5 @@ body:
   - type: input
     attributes:
       label: Color temperature range
-      description: In case this light supports color temperature, provide the color temperature range. [How to do this](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html#_3-1-retrieving-color-temperature-range-only-required-for-lights-which-support-color-temperature). This currently requires the latest [dev branch](https://www.zigbee2mqtt.io/advanced/more/switch-to-dev-branch.html).
-      placeholder: '{"colorTempPhysicalMin":153,"colorTempPhysicalMax":454}'
+      description: In case this light supports color temperature, provide the color temperature range. [How to do this](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html#_3-1-retrieving-color-temperature-range-only-required-for-lights-which-support-color-temperature).
+      placeholder: '{"colorTempPhysicalMin":153,"colorTempPhysicalMax":454}'

.github/ISSUE_TEMPLATE/problem_report.yaml

@@ -1,6 +1,5 @@
 name: Problem report
 description: Create a report to help us improve
-title: "[Problem]: "
 labels: [problem]
 body:
   - type: markdown
@@ -26,19 +25,19 @@ body:
       label: What did you expect to happen?
       placeholder: I expected that ...
     validations:
-      required: true
+      required: false
   - type: textarea
     id: reproduce
     attributes:
       label: How to reproduce it (minimal and precise)
       placeholder: First do this, than this..
     validations:
-      required: true
+      required: false
   - type: input
     id: z2m_version
     attributes:
       label: Zigbee2MQTT version
-      description: Can be found in the frontend -> settings -> about -> Zigbee2MQTT version
+      description: Can be found in the frontend -> settings -> about -> Zigbee2MQTT version. Are you running Zigbee2MQTT 1.18.1? Then read [this](https://github.com/Koenkk/zigbee2mqtt/releases/tag/1.19.0).
       placeholder: '1.22.1'
     validations:
       required: true
@@ -65,4 +64,4 @@ body:
       description: After enabling [debug logging](https://www.zigbee2mqtt.io/guide/usage/debug.html) the log can be found under `data/log`. Attach the file below
       placeholder: Click here and drag the file into it or click on "Attach files by.." below
     validations:
-      required: true
+      required: false

.github/workflows/ci.yml

@@ -7,10 +7,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v2
       with:
-        node-version: 14
-        registry-url: https://registry.npmjs.org/
+        node-version: 16
     - name: Get NodeJS Version
       id: nodeversion
       run: |

.github/workflows/update_deps.yml

@@ -14,7 +14,7 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 14
+        node-version: 16
     - run: rm -rf npm-shrinkwrap.json
     - run: npx npm-check-updates -u
     - run: npm install && npm shrinkwrap --dev

.github/workflows/update_frontend.yml

@@ -13,7 +13,7 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 14
+        node-version: 16
     - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
     - uses: peter-evans/create-pull-request@v3
       id: cpr

.github/workflows/update_zh.yml

@@ -13,7 +13,7 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 14
+        node-version: 16
     - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
     - uses: peter-evans/create-pull-request@v3
       id: cpr

.github/workflows/update_zhc.yml

@@ -13,7 +13,7 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 14
+        node-version: 16
     - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
     - uses: peter-evans/create-pull-request@v3
       id: cpr

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-alpine3.12 as base
+FROM node:16-alpine3.15 as base
 
 WORKDIR /app
 RUN apk add --no-cache tzdata eudev tini

index.js

@@ -47,6 +47,10 @@ async function build(reason) {
         exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
             if (err) {
                 process.stdout.write(', failed\n');
+                if (err.code === 134) {
+                    process.stderr.write(
+                        '\n\nBuild failed; ran out-of-memory, free some memory (RAM) and start again\n\n');
+                }
                 reject(err);
             } else {
                 process.stdout.write(', finished\n');

lib/controller.ts

@@ -104,13 +104,14 @@ class Controller {
             logger.error('Check https://www.zigbee2mqtt.io/guide/installation/20_zigbee2mqtt-fails-to-start.html for possible solutions'); /* eslint-disable-line max-len */
             logger.error('Exiting...');
             logger.error(error.stack);
-            this.exitCallback(1);
+            await this.exit(1);
         }
 
         // Disable some legacy options on new network creation
         if (startResult === 'reset') {
             settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
             settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['advanced', 'legacy_availability_payload'], false);
             settings.set(['device_options', 'legacy'], false);
             this.enableDisableExtension(false, 'BridgeLegacy');
         }
@@ -145,7 +146,7 @@ class Controller {
             logger.error(`MQTT failed to connect: ${error.message}`);
             logger.error('Exiting...');
             await this.zigbee.stop();
-            this.exitCallback(1);
+            await this.exit(1);
         }
 
         // Call extensions
@@ -197,13 +198,18 @@ class Controller {
         try {
             await this.zigbee.stop();
             logger.info('Stopped Zigbee2MQTT');
-            this.exitCallback(0);
+            await this.exit(0);
         } catch (error) {
             logger.error('Failed to stop Zigbee2MQTT');
-            this.exitCallback(1);
+            await this.exit(1);
         }
     }
 
+    async exit(code: number): Promise<void> {
+        await logger.end();
+        this.exitCallback(code);
+    }
+
     @bind async onZigbeeAdapterDisconnected(): Promise<void> {
         logger.error('Adapter disconnected, stopping');
         await this.stop();
@@ -222,11 +228,11 @@ class Controller {
         }
 
         const options: MQTTOptions = {
-            retain: utils.getObjectProperty(entity.settings, 'retain', false) as boolean,
-            qos: utils.getObjectProperty(entity.settings, 'qos', 0) as 0 | 1 | 2,
+            retain: utils.getObjectProperty(entity.options, 'retain', false) as boolean,
+            qos: utils.getObjectProperty(entity.options, 'qos', 0) as 0 | 1 | 2,
         };
 
-        const retention = utils.getObjectProperty(entity.settings, 'retention', false);
+        const retention = utils.getObjectProperty(entity.options, 'retention', false);
         if (retention !== false) {
             options.properties = {messageExpiryInterval: retention as number};
         }
@@ -259,12 +265,12 @@ class Controller {
         }
 
         // filter mqtt message attributes
-        if (entity.settings.filtered_attributes) {
-            entity.settings.filtered_attributes.forEach((a) => delete message[a]);
+        if (entity.options.filtered_attributes) {
+            entity.options.filtered_attributes.forEach((a) => delete message[a]);
         }
 
         if (Object.entries(message).length) {
-            const output = settings.get().experimental.output;
+            const output = settings.get().advanced.output;
             if (output === 'attribute_and_json' || output === 'json') {
                 await this.mqtt.publish(entity.name, stringify(message), options);
             }

lib/eventBus.ts

@@ -27,10 +27,17 @@ export default class EventBus {
         this.on('permitJoinChanged', callback, key);
     }
 
-    public emitDeviceRenamed(data: eventdata.DeviceRenamed): void {
+    public emitPublishAvailability(): void {
+        this.emitter.emit('publishAvailability');
+    }
+    public onPublishAvailability(key: ListenerKey, callback: () => void): void {
+        this.on('publishAvailability', callback, key);
+    }
+
+    public emitEntityRenamed(data: eventdata.EntityRenamed): void {
         this.emitter.emit('deviceRenamed', data);
     }
-    public onDeviceRenamed(key: ListenerKey, callback: (data: eventdata.DeviceRenamed) => void): void {
+    public onEntityRenamed(key: ListenerKey, callback: (data: eventdata.EntityRenamed) => void): void {
         this.on('deviceRenamed', callback, key);
     }
 

lib/extension/availability.ts

@@ -5,6 +5,12 @@ import * as settings from '../util/settings';
 import debounce from 'debounce';
 import bind from 'bind-decorator';
 
+const retrieveOnReconnect = [
+    {keys: ['state']},
+    {keys: ['brightness'], condition: (state: KeyValue): boolean => state.state === 'ON'},
+    {keys: ['color', 'color_temp'], condition: (state: KeyValue): boolean => state.state === 'ON'},
+];
+
 export default class Availability extends Extension {
     private timers: {[s: string]: NodeJS.Timeout} = {};
     private availabilityCache: {[s: string]: boolean} = {};
@@ -13,17 +19,14 @@ export default class Availability extends Extension {
     private pingQueueExecuting = false;
 
     private getTimeout(device: Device): number {
-        if (typeof device.settings.availability === 'object' && device.settings.availability?.timeout != null) {
-            return utils.minutes(device.settings.availability.timeout);
+        if (typeof device.options.availability === 'object' && device.options.availability?.timeout != null) {
+            return utils.minutes(device.options.availability.timeout);
         }
 
         const key = this.isActiveDevice(device) ? 'active' : 'passive';
-        const availabilitySettings = settings.get().availability;
-        if (typeof availabilitySettings === 'object' && availabilitySettings[key]?.timeout != null) {
-            return utils.minutes(availabilitySettings[key]?.timeout);
-        }
-
-        return key === 'active' ? utils.minutes(10) : utils.hours(25);
+        let value = settings.get().availability?.[key]?.timeout;
+        if (value == null) value = key == 'active' ? 10 : 1500;
+        return utils.minutes(value);
     }
 
     private isActiveDevice(device: Device): boolean {
@@ -31,9 +34,14 @@ export default class Availability extends Extension {
             device.zh.powerSource === 'Mains (single phase)';
     }
 
-    private isAvailable(device: Device): boolean {
-        const ago = Date.now() - device.zh.lastSeen;
-        return ago < this.getTimeout(device);
+    private isAvailable(entity: Device | Group): boolean {
+        if (entity.isDevice()) {
+            const ago = Date.now() - entity.zh.lastSeen;
+            return ago < this.getTimeout(entity);
+        } else {
+            return entity.membersDevices().length === 0 ||
+                entity.membersDevices().map((d) => this.availabilityCache[d.ieeeAddr]).includes(true);
+        }
     }
 
     private resetTimer(device: Device): void {
@@ -95,58 +103,78 @@ export default class Availability extends Extension {
     }
 
     override async start(): Promise<void> {
-        this.eventBus.onDeviceRenamed(this, (data) => this.publishAvailability(data.device, false, true));
+        this.eventBus.onEntityRenamed(this, (data) => {
+            if (utils.isAvailabilityEnabledForEntity(data.entity, settings.get())) {
+                this.mqtt.publish(`${data.from}/availability`, null, {retain: true, qos: 0});
+                this.publishAvailability(data.entity, false, true);
+            }
+        });
+
         this.eventBus.onDeviceRemoved(this, (data) => clearTimeout(this.timers[data.ieeeAddr]));
         this.eventBus.onDeviceLeave(this, (data) => clearTimeout(this.timers[data.ieeeAddr]));
         this.eventBus.onDeviceAnnounce(this, (data) => this.retrieveState(data.device));
         this.eventBus.onLastSeenChanged(this, this.onLastSeenChanged);
+        this.eventBus.onPublishAvailability(this, this.publishAvailabilityForAllEntities);
+        this.eventBus.onGroupMembersChanged(this, (data) => this.publishAvailability(data.group, false));
+        this.publishAvailabilityForAllEntities();
+    }
 
-        for (const device of this.zigbee.devices(false)) {
-            if (utils.isAvailabilityEnabledForDevice(device, settings.get())) {
+    @bind private publishAvailabilityForAllEntities(): void {
+        for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+            if (utils.isAvailabilityEnabledForEntity(entity, settings.get())) {
                 // Publish initial availablility
-                this.publishAvailability(device, true);
+                this.publishAvailability(entity, true, false, true);
 
-                this.resetTimer(device);
+                if (entity.isDevice()) {
+                    this.resetTimer(entity);
 
-                // If an active device is initially unavailable, ping it.
-                if (this.isActiveDevice(device) && !this.isAvailable(device)) {
-                    this.addToPingQueue(device);
+                    // If an active device is initially unavailable, ping it.
+                    if (this.isActiveDevice(entity) && !this.isAvailable(entity)) {
+                        this.addToPingQueue(entity);
+                    }
                 }
             }
         }
     }
 
-    private publishAvailability(device: Device, logLastSeen: boolean, forcePublish=false): void {
-        if (logLastSeen) {
-            const ago = Date.now() - device.zh.lastSeen;
-            if (this.isActiveDevice(device)) {
-                logger.debug(`Active device '${device.name}' was last seen ` +
+    private publishAvailability(entity: Device | Group, logLastSeen: boolean,
+        forcePublish=false, skipGroups=false): void {
+        if (logLastSeen && entity.isDevice()) {
+            const ago = Date.now() - entity.zh.lastSeen;
+            if (this.isActiveDevice(entity)) {
+                logger.debug(`Active device '${entity.name}' was last seen ` +
                     `'${(ago / utils.minutes(1)).toFixed(2)}' minutes ago.`);
             } else {
                 logger.debug(
-                    `Passive device '${device.name}' was last seen '${(ago / utils.hours(1)).toFixed(2)}' hours ago.`);
+                    `Passive device '${entity.name}' was last seen '${(ago / utils.hours(1)).toFixed(2)}' hours ago.`);
             }
         }
 
-        const available = this.isAvailable(device);
-        if (!forcePublish && this.availabilityCache[device.ieeeAddr] == available) {
+        const available = this.isAvailable(entity);
+        if (!forcePublish && this.availabilityCache[entity.ID] == available) {
             return;
         }
 
-        if (device.ieeeAddr in this.availabilityCache && available &&
-            this.availabilityCache[device.ieeeAddr] === false) {
-            logger.debug(`Device '${device.name}' reconnected`);
-            this.retrieveState(device);
+        if (entity.isDevice() && entity.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[entity.ieeeAddr] === false) {
+            logger.debug(`Device '${entity.name}' reconnected`);
+            this.retrieveState(entity);
         }
 
-        const topic = `${device.name}/availability`;
-        const payload = available ? 'online' : 'offline';
-        this.availabilityCache[device.ieeeAddr] = available;
+        const topic = `${entity.name}/availability`;
+        const payload = utils.availabilityPayload(available ? 'online' : 'offline', settings.get());
+        this.availabilityCache[entity.ID] = available;
         this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+
+        if (!skipGroups && entity.isDevice()) {
+            this.zigbee.groups().filter((g) => g.hasMember(entity))
+                .filter((g) => utils.isAvailabilityEnabledForEntity(g, settings.get()))
+                .forEach((g) => this.publishAvailability(g, false, forcePublish));
+        }
     }
 
     @bind private onLastSeenChanged(data: eventdata.LastSeenChanged): void {
-        if (utils.isAvailabilityEnabledForDevice(data.device, settings.get())) {
+        if (utils.isAvailabilityEnabledForEntity(data.device, settings.get())) {
             // Remove from ping queue, not necessary anymore since we know the device is online.
             this.removeFromPingQueue(data.device);
             this.resetTimer(data.device);
@@ -165,17 +193,18 @@ export default class Availability extends Extension {
          * device can send multiple times after each other.
          */
         if (device.definition && !device.zh.interviewing && !this.retrieveStateDebouncers[device.ieeeAddr]) {
-            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(() => {
+            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(async () => {
                 logger.debug(`Retrieving state of '${device.name}' after reconnect`);
                 // Color and color temperature converters do both, only needs to be called once.
-                const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
-                for (const keys of keySet) {
-                    const converter = device.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
-                    converter?.convertGet?.(device.endpoint(), keys[0],
+                for (const item of retrieveOnReconnect) {
+                    if (item.condition && this.state.get(device) && !item.condition(this.state.get(device))) continue;
+                    const converter = device.definition.toZigbee.find((c) => c.key.find((k) => item.keys.includes(k)));
+                    await converter?.convertGet?.(device.endpoint(), item.keys[0],
                         {message: this.state.get(device) || {}, mapped: device.definition})
                         .catch((e) => {
                             logger.error(`Failed to read state of '${device.name}' after reconnect (${e.message})`);
                         });
+                    await utils.sleep(500);
                 }
             }, utils.seconds(2));
         }

lib/extension/bridge.ts

@@ -64,7 +64,7 @@ export default class Bridge extends Extension {
         this.zigbee2mqttVersion = await utils.getZigbee2MQTTVersion();
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
-        this.eventBus.onDeviceRenamed(this, () => this.publishInfo());
+        this.eventBus.onEntityRenamed(this, () => this.publishInfo());
         this.eventBus.onGroupMembersChanged(this, () => this.publishGroups());
         this.eventBus.onDevicesChanged(this, () => this.publishDevices() && this.publishInfo());
         this.eventBus.onPermitJoinChanged(this, () => !this.zigbee.isStopping() && this.publishInfo());
@@ -392,9 +392,9 @@ export default class Bridge extends Extension {
 
         const ID = message.id;
         const entity = this.getEntity(entityType, ID);
-        const oldOptions = objectAssignDeep({}, cleanup(entity.settings));
+        const oldOptions = objectAssignDeep({}, cleanup(entity.options));
         settings.changeEntityOptions(ID, message.options);
-        const newOptions = cleanup(entity.settings);
+        const newOptions = cleanup(entity.options);
         await this.publishInfo();
 
         logger.info(`Changed config for ${entityType} ${ID}`);
@@ -419,7 +419,7 @@ export default class Bridge extends Extension {
         await endpoint.configureReporting(message.cluster, [{
             attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
             maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
-        }]);
+        }], message.options);
 
         this.publishDevices();
 
@@ -448,16 +448,17 @@ export default class Bridge extends Extension {
         const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
             message.homeassistant_rename : false;
         const entity = this.getEntity(entityType, from);
-        const oldFriendlyName = entity.settings.friendly_name;
+        const oldFriendlyName = entity.options.friendly_name;
 
         settings.changeFriendlyName(from, to);
 
         // Clear retained messages
         this.mqtt.publish(oldFriendlyName, '', {retain: true});
 
+        this.eventBus.emitEntityRenamed({entity: entity, homeAssisantRename, from: oldFriendlyName, to});
+
         if (entity instanceof Device) {
             this.publishDevices();
-            this.eventBus.emitDeviceRenamed({device: entity, homeAssisantRename, from: oldFriendlyName, to});
         } else {
             this.publishGroups();
             this.publishInfo();
@@ -649,6 +650,7 @@ export default class Bridge extends Extension {
                 network_address: device.zh.networkAddress,
                 supported: !!device.definition,
                 friendly_name: device.name,
+                description: device.options.description,
                 definition: this.getDefinitionPayload(device),
                 power_source: device.zh.powerSource,
                 software_build_id: device.zh.softwareBuildID,
@@ -670,6 +672,7 @@ export default class Bridge extends Extension {
             return {
                 id: g.ID,
                 friendly_name: g.ID === 901 ? 'default_bind_group' : g.name,
+                description: g.options.description,
                 scenes: this.getScenes(g.zh),
                 members: g.zh.members.map((e) => {
                     return {ieee_address: e.getDevice().ieeeAddr, endpoint: e.ID};
@@ -682,7 +685,7 @@ export default class Bridge extends Extension {
 
     getDefinitionPayload(device: Device): DefinitionPayload {
         if (!device.definition) return null;
-        let icon = device.settings.icon ? device.settings.icon : device.definition.icon;
+        let icon = device.options.icon ? device.options.icon : device.definition.icon;
         if (icon) {
             icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(device.zh.modelID));
             icon = icon.replace('${model}', utils.sanitizeImageParameter(device.definition.model));

lib/extension/configure.ts

@@ -85,7 +85,7 @@ export default class Configure extends Extension {
     private async configure(device: Device, event: 'started' | 'zigbee_event' | 'reporting_disabled' | 'mqtt_message',
         force=false, thowError=false): Promise<void> {
         if (!force) {
-            if (!device.definition?.configure || device.zh.interviewing) {
+            if (!device.definition?.configure || !device.zh.interviewCompleted) {
                 return;
             }
 
@@ -112,7 +112,8 @@ export default class Configure extends Extension {
 
         logger.info(`Configuring '${device.name}'`);
         try {
-            await device.definition.configure(device.zh, this.zigbee.firstCoordinatorEndpoint(), logger);
+            await device.definition.configure(device.zh, this.zigbee.firstCoordinatorEndpoint(), logger,
+                device.options);
             logger.info(`Successfully configured '${device.name}'`);
             device.zh.meta.configured = zhc.getConfigureKey(device.definition);
             device.zh.save();

lib/extension/frontend.ts

@@ -17,9 +17,9 @@ import bind from 'bind-decorator';
  */
 export default class Frontend extends Extension {
     private mqttBaseTopic = settings.get().mqtt.base_topic;
-    private host = settings.get().frontend.host || '0.0.0.0';
-    private port = settings.get().frontend.port || 8080;
-    private authToken = settings.get().frontend.auth_token || false;
+    private host = settings.get().frontend.host;
+    private port = settings.get().frontend.port;
+    private authToken = settings.get().frontend.auth_token;
     private retainedMessages = new Map();
     private server: http.Server;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any

lib/extension/groups.ts

@@ -123,7 +123,7 @@ export default class Groups extends Extension {
         if (Object.keys(payload).length) {
             const entity = data.entity;
             const groups = this.zigbee.groups().filter((g) => {
-                return g.settings && (!g.settings.hasOwnProperty('optimistic') || g.settings.optimistic);
+                return g.options && (!g.options.hasOwnProperty('optimistic') || g.options.optimistic);
             });
 
             if (entity instanceof Device) {

lib/extension/homeassistant.ts

@@ -65,16 +65,16 @@ export default class HomeAssistant extends Extension {
     private discovered: {[s: string]:
         {topics: Set<string>, mockProperties: Set<MockProperty>, objectIDs: Set<string>}} = {};
     private discoveredTriggers : {[s: string]: Set<string>}= {};
-    private discoveryTopic = settings.get().advanced.homeassistant_discovery_topic;
-    private statusTopic = settings.get().advanced.homeassistant_status_topic;
-    private entityAttributes = settings.get().advanced.homeassistant_legacy_entity_attributes;
+    private discoveryTopic = settings.get().homeassistant.discovery_topic;
+    private statusTopic = settings.get().homeassistant.status_topic;
+    private entityAttributes = settings.get().homeassistant.legacy_entity_attributes;
     private zigbee2MQTTVersion: string;
 
     constructor(zigbee: Zigbee, mqtt: MQTT, state: State, publishEntityState: PublishEntityState,
         eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
         restartCallback: () => void, addExtension: (extension: Extension) => Promise<void>) {
         super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);
-        if (settings.get().experimental.output === 'attribute') {
+        if (settings.get().advanced.output === 'attribute') {
             throw new Error('Home Assistant integration is not possible with attribute output!');
         }
     }
@@ -88,13 +88,14 @@ export default class HomeAssistant extends Extension {
 
         this.eventBus.onDeviceRemoved(this, this.onDeviceRemoved);
         this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
-        this.eventBus.onDeviceRenamed(this, this.onDeviceRenamed);
+        this.eventBus.onEntityRenamed(this, this.onEntityRenamed);
         this.eventBus.onPublishEntityState(this, this.onPublishEntityState);
         this.eventBus.onGroupMembersChanged(this, this.onGroupMembersChanged);
         this.eventBus.onDeviceAnnounce(this, this.onZigbeeEvent);
         this.eventBus.onDeviceJoined(this, this.onZigbeeEvent);
         this.eventBus.onDeviceInterview(this, this.onZigbeeEvent);
         this.eventBus.onDeviceMessage(this, this.onZigbeeEvent);
+        this.eventBus.onEntityOptionsChanged(this, (data) => this.discover(data.entity, true));
 
         this.mqtt.subscribe(this.statusTopic);
         this.mqtt.subscribe(defaultStatusTopic);
@@ -104,6 +105,9 @@ export default class HomeAssistant extends Extension {
         for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
             this.discover(entity, true);
         }
+
+        // Send availability messages, this is required if the legacy_availability_payload option has been changed.
+        this.eventBus.emitPublishAvailability();
     }
 
     private exposeToConfig(exposes: zhc.DefinitionExpose[], entityType: 'device' | 'group',
@@ -127,6 +131,11 @@ export default class HomeAssistant extends Extension {
             const hasBrightness = exposes.find((expose) => expose.features.find((e) => e.name === 'brightness'));
             const hasColorTemp = exposes.find((expose) => expose.features.find((e) => e.name === 'color_temp'));
             const state = firstExpose.features.find((f) => f.name === 'state');
+            // Prefer HS over XY when at least one of the lights in the group prefers HS over XY.
+            // A light prefers HS over XY when HS is earlier in the feature array than HS.
+            const preferHS = exposes.map((e) => [e.features.findIndex((ee) => ee.name === 'color_xy'),
+                e.features.findIndex((ee) => ee.name === 'color_hs')])
+                .filter((d) => d[0] !== -1 && d[1] !== -1 && d[1] < d[0]).length !== 0;
 
             const discoveryEntry: DiscoveryEntry = {
                 type: 'light',
@@ -143,8 +152,8 @@ export default class HomeAssistant extends Extension {
             };
 
             const colorModes = [
-                hasColorXY ? 'xy' : null,
-                !hasColorXY && hasColorHS ? 'hs' : null,
+                hasColorXY && !preferHS ? 'xy' : null,
+                (!hasColorXY || preferHS) && hasColorHS ? 'hs' : null,
                 hasColorTemp ? 'color_temp' : null,
             ].filter((c) => c);
 
@@ -371,6 +380,8 @@ export default class HomeAssistant extends Extension {
         } else if (firstExpose.type === 'cover') {
             const position = exposes.find((expose) => expose.features.find((e) => e.name === 'position'));
             const tilt = exposes.find((expose) => expose.features.find((e) => e.name === 'tilt'));
+            const motorState = definitionExposes?.find((e) => e.type === 'enum' && e.name === 'motor_state' &&
+                e.access === ACCESS_STATE);
 
             const discoveryEntry: DiscoveryEntry = {
                 type: 'cover',
@@ -389,6 +400,43 @@ export default class HomeAssistant extends Extension {
                 discoveryEntry.discovery_payload.command_topic = true;
                 discoveryEntry.discovery_payload.state_topic = !position;
                 discoveryEntry.discovery_payload.command_topic_prefix = endpoint;
+
+                // For curtains that have `motor_state` lookup a possible state names and make this
+                // available for discovery. If the curtains only support the `running` value,
+                // then we use it anyway. The movement direction is calculated (assumed) in this case.
+                if (motorState) {
+                    const openingLookup = ['opening', 'open', 'forward', 'up', 'rising'];
+                    const closingLookup = ['closing', 'close', 'backward', 'back', 'reverse', 'down', 'declining'];
+                    const stoppedLookup = ['stopped', 'stop', 'pause', 'paused'];
+
+                    const openingState = motorState.values.find((s) => openingLookup.includes(s.toLowerCase()));
+                    const closingState = motorState.values.find((s) => closingLookup.includes(s.toLowerCase()));
+                    const stoppedState = motorState.values.find((s) => stoppedLookup.includes(s.toLowerCase()));
+
+                    if (openingState) {
+                        discoveryEntry.discovery_payload.state_opening = openingState;
+                    }
+                    if (closingState) {
+                        discoveryEntry.discovery_payload.state_closing = closingState;
+                    }
+                    if (stoppedState) {
+                        discoveryEntry.discovery_payload.state_stopped = stoppedState;
+                    }
+
+                    if (openingState || closingState || stoppedState) {
+                        discoveryEntry.discovery_payload.state_topic = true;
+                        discoveryEntry.discovery_payload.value_template = `{{ value_json.motor_state }}`;
+                    }
+                } else {
+                    const running = definitionExposes?.find((e) => e.type === 'binary' && e.name === 'running');
+
+                    if (running) {
+                        discoveryEntry.discovery_payload.state_topic = true;
+                        discoveryEntry.discovery_payload.value_template = `{% if not value_json.running %} ` +
+                            `stopped {% else %} {% if value_json.position > 0 %} closing {% else %} ` +
+                            `opening {% endif %} {% endif %}`;
+                    }
+                }
             }
 
             if (!position && !tilt) {
@@ -490,6 +538,7 @@ export default class HomeAssistant extends Extension {
                 color_sync: {entity_category: 'config', icon: 'mdi:sync-circle'},
                 consumer_connected: {entity_category: 'diagnostic', device_class: 'connectivity'},
                 contact: {device_class: 'door'},
+                garage_door_contact: {device_class: 'garage'},
                 eco_mode: {entity_category: 'config', icon: 'mdi:leaf'},
                 expose_pin: {entity_category: 'config', icon: 'mdi:pin'},
                 gas: {device_class: 'gas'},
@@ -552,6 +601,12 @@ export default class HomeAssistant extends Extension {
             }
         } else if (firstExpose.type === 'numeric') {
             const lookup: {[s: string]: KeyValue} = {
+                ac_frequency: {device_class: 'frequency', enabled_by_default: false, entity_category: 'diagnostic',
+                    state_class: 'measurement'},
+                alarm_humidity_max: {device_class: 'humidity', icon: 'mdi:water-plus'},
+                alarm_humidity_min: {device_class: 'humidity', icon: 'mdi:water-minus'},
+                alarm_temperature_max: {device_class: 'temperature', icon: 'mdi:thermometer-high'},
+                alarm_temperature_min: {device_class: 'temperature', icon: 'mdi:thermometer-low'},
                 angle: {icon: 'angle-acute'},
                 angle_axis: {icon: 'angle-acute'},
                 aqi: {device_class: 'aqi', state_class: 'measurement'},
@@ -613,8 +668,11 @@ export default class HomeAssistant extends Extension {
                 occupancy_timeout: {entity_category: 'config', icon: 'mdi:timer'},
                 pm10: {device_class: 'pm10', state_class: 'measurement'},
                 pm25: {device_class: 'pm25', state_class: 'measurement'},
+                people: {state_class: 'measurement', icon: 'mdi:account-multiple'},
                 position: {icon: 'mdi:valve', state_class: 'measurement'},
                 power: {device_class: 'power', entity_category: 'diagnostic', state_class: 'measurement'},
+                power_factor: {device_class: 'power_factor', enabled_by_default: false,
+                    entity_category: 'diagnostic', state_class: 'measurement'},
                 precision: {entity_category: 'config', icon: 'mdi:decimal-comma-increase'},
                 pressure: {device_class: 'pressure', state_class: 'measurement'},
                 presence_timeout: {entity_category: 'config', icon: 'mdi:timer'},
@@ -653,6 +711,13 @@ export default class HomeAssistant extends Extension {
                 z_axis: {icon: 'mdi:axis-z-arrow'},
             };
 
+            const extraAttrs = {};
+
+            // If a variable includes Wh, mark it as energy
+            if (firstExpose.unit && ['Wh', 'kWh'].includes(firstExpose.unit)) {
+                Object.assign(extraAttrs, {device_class: 'energy', state_class: 'total_increasing'});
+            }
+
             const allowsSet = firstExpose.access & ACCESS_SET;
 
             const discoveryEntry: DiscoveryEntry = {
@@ -664,13 +729,14 @@ export default class HomeAssistant extends Extension {
                     enabled_by_default: !allowsSet,
                     ...(firstExpose.unit && {unit_of_measurement: firstExpose.unit}),
                     ...lookup[firstExpose.name],
+                    ...extraAttrs,
                 },
             };
             discoveryEntries.push(discoveryEntry);
 
             /**
              * If numeric attribute has SET access then expose as SELECT entity too.
-             * Note: currently both sensor and number are discoverd, this is to avoid
+             * Note: currently both sensor and number are discovered, this is to avoid
              * breaking changes for sensors already existing in HA (legacy).
              */
             if (allowsSet) {
@@ -696,6 +762,8 @@ export default class HomeAssistant extends Extension {
         } else if (firstExpose.type === 'enum') {
             const lookup: {[s: string]: KeyValue} = {
                 action: {icon: 'mdi:gesture-double-tap'},
+                alarm_humidity: {icon: 'mdi:water-percent-alert'},
+                alarm_temperature: {icon: 'mdi:thermometer-alert'},
                 backlight_auto_dim: {entity_category: 'config', icon: 'mdi:brightness-auto'},
                 backlight_mode: {entity_category: 'config', icon: 'mdi:lightbulb'},
                 color_power_on_behavior: {entity_category: 'config', icon: 'mdi:palette'},
@@ -714,6 +782,7 @@ export default class HomeAssistant extends Extension {
                 sensitivity: {entity_category: 'config', icon: 'mdi:tune'},
                 sensors_type: {entity_category: 'config', icon: 'mdi:tune'},
                 sound_volume: {entity_category: 'config', icon: 'mdi:volume-high'},
+                status: {icon: 'mdi:state-machine'},
                 switch_type: {entity_category: 'config', icon: 'mdi:tune'},
                 thermostat_unit: {entity_category: 'config', icon: 'mdi:thermometer'},
                 volume: {entity_category: 'config', icon: 'mdi: volume-high'},
@@ -734,7 +803,7 @@ export default class HomeAssistant extends Extension {
 
                 /**
                  * If enum attribute has SET access then expose as SELECT entity too.
-                 * Note: currently both sensor and select are discoverd, this is to avoid
+                 * Note: currently both sensor and select are discovered, this is to avoid
                  * breaking changes for sensors already existing in HA (legacy).
                  */
                 if ((firstExpose.access & ACCESS_SET)) {
@@ -827,7 +896,7 @@ export default class HomeAssistant extends Extension {
          * can use Home Assistant entities in automations.
          * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
          */
-        if (settings.get().advanced.homeassistant_legacy_triggers) {
+        if (settings.get().homeassistant.legacy_triggers) {
             const keys = ['action', 'click'].filter((k) => data.message[k]);
             for (const key of keys) {
                 this.publishEntityState(data.entity, {[key]: ''});
@@ -850,25 +919,25 @@ export default class HomeAssistant extends Extension {
         }
     }
 
-    @bind onDeviceRenamed(data: eventdata.DeviceRenamed): void {
-        logger.debug(`Refreshing Home Assistant discovery topic for '${data.device.ieeeAddr}'`);
+    @bind onEntityRenamed(data: eventdata.EntityRenamed): void {
+        logger.debug(`Refreshing Home Assistant discovery topic for '${data.entity.name}'`);
 
         // Clear before rename so Home Assistant uses new friendly_name
         // https://github.com/Koenkk/zigbee2mqtt/issues/4096#issuecomment-674044916
         if (data.homeAssisantRename) {
-            for (const config of this.getConfigs(data.device)) {
-                const topic = this.getDiscoveryTopic(config, data.device);
+            for (const config of this.getConfigs(data.entity)) {
+                const topic = this.getDiscoveryTopic(config, data.entity);
                 this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
             }
         }
 
-        this.discover(data.device, true);
+        this.discover(data.entity, true);
 
-        if (this.discoveredTriggers[data.device.ieeeAddr]) {
-            for (const config of this.discoveredTriggers[data.device.ieeeAddr]) {
+        if (data.entity.isDevice() && this.discoveredTriggers[data.entity.ieeeAddr]) {
+            for (const config of this.discoveredTriggers[data.entity.ieeeAddr]) {
                 const key = config.substring(0, config.indexOf('_'));
                 const value = config.substring(config.indexOf('_') + 1);
-                this.publishDeviceTriggerDiscover(data.device, key, value, true);
+                this.publishDeviceTriggerDiscover(data.entity, key, value, true);
             }
         }
     }
@@ -880,8 +949,9 @@ export default class HomeAssistant extends Extension {
 
         let configs: DiscoveryEntry[] = [];
         if (isDevice) {
-            for (const expose of entity.exposes()) {
-                configs.push(...this.exposeToConfig([expose], 'device', entity.definition, entity.exposes()));
+            const exposes = entity.exposes(); // avoid calling it hundred of times/s
+            for (const expose of exposes) {
+                configs.push(...this.exposeToConfig([expose], 'device', entity.definition, exposes));
             }
 
             for (const mapping of legacyMapping) {
@@ -970,19 +1040,19 @@ export default class HomeAssistant extends Extension {
             configs.push(updateAvailableSensor);
         }
 
-        if (isDevice && entity.settings.hasOwnProperty('legacy') && !entity.settings.legacy) {
+        if (isDevice && entity.options.hasOwnProperty('legacy') && !entity.options.legacy) {
             configs = configs.filter((c) => c !== sensorClick);
         }
 
-        if (!settings.get().advanced.homeassistant_legacy_triggers) {
+        if (!settings.get().homeassistant.legacy_triggers) {
             configs = configs.filter((c) => c.object_id !== 'action' && c.object_id !== 'click');
         }
 
         // deep clone of the config objects
         configs = JSON.parse(JSON.stringify(configs));
 
-        if (entity.settings.homeassistant) {
-            const s = entity.settings.homeassistant;
+        if (entity.options.homeassistant) {
+            const s = entity.options.homeassistant;
             configs = configs.filter((config) => !s.hasOwnProperty(config.object_id) || s[config.object_id] != null);
             configs.forEach((config) => {
                 const configOverride = s[config.object_id];
@@ -1001,14 +1071,14 @@ export default class HomeAssistant extends Extension {
     }
 
     private discover(entity: Device | Group, force=false): void {
-        // Check if already discoverd and check if there are configs.
+        // Check if already discovered and check if there are configs.
         const discoverKey = this.getDiscoverKey(entity);
         const discover = force || !this.discovered[discoverKey];
 
         if (entity.isGroup()) {
             if (!discover || entity.zh.members.length === 0) return;
         } else if (!discover || !entity.definition || entity.zh.interviewing ||
-            (entity.settings.hasOwnProperty('homeassistant') && !entity.settings.homeassistant)) {
+            (entity.options.hasOwnProperty('homeassistant') && !entity.options.homeassistant)) {
             return;
         }
 
@@ -1043,9 +1113,11 @@ export default class HomeAssistant extends Extension {
                 payload.json_attributes_topic = stateTopic;
             }
 
-            // Set (unique) name, separate by space if friendlyName contains space.
-            const nameSeparator = entity.name.includes('_') ? '_' : ' ';
-            payload.name = entity.name;
+            const devicePayload = this.getDevicePayload(entity);
+
+            // Set (unique) name, separate by space if device name contains space.
+            const nameSeparator = devicePayload.name.includes('_') ? '_' : ' ';
+            payload.name = devicePayload.name;
             if (config.object_id.startsWith(config.type) && config.object_id.includes('_')) {
                 payload.name += `${nameSeparator}${config.object_id.split(/_(.+)/)[1]}`;
             } else if (!config.object_id.startsWith(config.type)) {
@@ -1053,22 +1125,24 @@ export default class HomeAssistant extends Extension {
             }
 
             // Set unique_id
-            payload.unique_id = `${entity.settings.ID}_${config.object_id}_${settings.get().mqtt.base_topic}`;
+            payload.unique_id = `${entity.options.ID}_${config.object_id}_${settings.get().mqtt.base_topic}`;
 
             // Attributes for device registry
-            payload.device = this.getDevicePayload(entity);
+            payload.device = devicePayload;
 
             // Availability payload
             payload.availability = [{topic: `${settings.get().mqtt.base_topic}/bridge/state`}];
 
-            const availabilityEnabled =
-                entity.isDevice() && utils.isAvailabilityEnabledForDevice(entity, settings.get());
             /* istanbul ignore next */
-            if (availabilityEnabled) {
+            if (utils.isAvailabilityEnabledForEntity(entity, settings.get())) {
                 payload.availability_mode = 'all';
                 payload.availability.push({topic: `${baseTopic}/availability`});
             }
 
+            if (!settings.get().advanced.legacy_availability_payload) {
+                payload.availability.forEach((a: KeyValue) => a.value_template = '{{ value_json.state }}');
+            }
+
             const commandTopicPrefix = payload.command_topic_prefix ? `${payload.command_topic_prefix}/` : '';
             delete payload.command_topic_prefix;
             const commandTopicPostfix = payload.command_topic_postfix ? `/${payload.command_topic_postfix}` : '';
@@ -1171,11 +1245,13 @@ export default class HomeAssistant extends Extension {
             }
 
             // Override configuration with user settings.
-            if (entity.settings.hasOwnProperty('homeassistant')) {
-                const add = (obj: KeyValue): void => {
+            if (entity.options.hasOwnProperty('homeassistant')) {
+                const add = (obj: KeyValue, ignoreName: boolean): void => {
                     Object.keys(obj).forEach((key) => {
                         if (['type', 'object_id'].includes(key)) {
                             return;
+                        } else if (ignoreName && key === 'name') {
+                            return;
                         } else if (['number', 'string', 'boolean'].includes(typeof obj[key]) ||
                             Array.isArray(obj[key])) {
                             payload[key] = obj[key];
@@ -1189,10 +1265,10 @@ export default class HomeAssistant extends Extension {
                     });
                 };
 
-                add(entity.settings.homeassistant);
+                add(entity.options.homeassistant, true);
 
-                if (entity.settings.homeassistant.hasOwnProperty(config.object_id)) {
-                    add(entity.settings.homeassistant[config.object_id]);
+                if (entity.options.homeassistant.hasOwnProperty(config.object_id)) {
+                    add(entity.options.homeassistant[config.object_id], false);
                 }
             }
 
@@ -1210,7 +1286,7 @@ export default class HomeAssistant extends Extension {
         const discoveryMatch = data.topic.match(discoveryRegex);
         const isDeviceAutomation = discoveryMatch && discoveryMatch[1] === 'device_automation';
         if (discoveryMatch) {
-            // Clear outdated discovery configs and remember already discoverd device_automations
+            // Clear outdated discovery configs and remember already discovered device_automations
             let message: KeyValue = null;
             try {
                 message = JSON.parse(data.message);
@@ -1252,7 +1328,7 @@ export default class HomeAssistant extends Extension {
                     `${this.discoveryTopic}/${this.getDiscoveryTopic(c, entity)}` === data.topic);
             }
             // Device was flagged to be excluded from homeassistant discovery
-            clear = clear || (entity.settings.hasOwnProperty('homeassistant') && !entity.settings.homeassistant);
+            clear = clear || (entity.options.hasOwnProperty('homeassistant') && !entity.options.homeassistant);
 
             if (clear) {
                 logger.debug(`Clearing Home Assistant config '${data.topic}'`);
@@ -1281,9 +1357,16 @@ export default class HomeAssistant extends Extension {
     private getDevicePayload(entity: Device | Group): KeyValue {
         const identifierPostfix = entity.isGroup() ?
             `zigbee2mqtt_${this.getEncodedBaseTopic()}` : 'zigbee2mqtt';
+
+        // Allow device name to be overriden by homeassistant config
+        let deviceName = entity.name;
+        if (typeof entity.options.homeassistant?.name === 'string') {
+            deviceName = entity.options.homeassistant.name;
+        }
+
         const payload: KeyValue = {
-            identifiers: [`${identifierPostfix}_${entity.settings.ID}`],
-            name: entity.name,
+            identifiers: [`${identifierPostfix}_${entity.options.ID}`],
+            name: deviceName,
             sw_version: `Zigbee2MQTT ${this.zigbee2MQTTVersion}`,
         };
 
@@ -1331,8 +1414,8 @@ export default class HomeAssistant extends Extension {
     }
 
     private async publishDeviceTriggerDiscover(device: Device, key: string, value: string, force=false): Promise<void> {
-        const haConfig = device.settings.homeassistant;
-        if (device.settings.hasOwnProperty('homeassistant') && (haConfig == null ||
+        const haConfig = device.options.homeassistant;
+        if (device.options.hasOwnProperty('homeassistant') && (haConfig == null ||
                 (haConfig.hasOwnProperty('device_automation') && typeof haConfig === 'object' &&
                     haConfig.device_automation == null))) {
             return;

lib/extension/legacy/bridgeLegacy.ts

@@ -51,7 +51,7 @@ export default class BridgeLegacy extends Extension {
         try {
             const entity = settings.getDevice(message);
             assert(entity, `Entity '${message}' does not exist`);
-            settings.whitelistDevice(entity.ID.toString());
+            settings.addDeviceToPasslist(entity.ID.toString());
             logger.info(`Whitelisted '${entity.friendly_name}'`);
             this.mqtt.publish(
                 'bridge/log',
@@ -217,7 +217,7 @@ export default class BridgeLegacy extends Extension {
             logger.info(`Successfully renamed - ${from} to ${to} `);
             const entity = this.zigbee.resolveEntity(to);
             if (entity.isDevice()) {
-                this.eventBus.emitDeviceRenamed({homeAssisantRename: false, from, to, device: entity});
+                this.eventBus.emitEntityRenamed({homeAssisantRename: false, from, to, entity});
             }
 
             this.mqtt.publish(
@@ -336,7 +336,7 @@ export default class BridgeLegacy extends Extension {
         }
 
         if (action === 'ban') {
-            settings.banDevice(ieeeAddr);
+            settings.blockDevice(ieeeAddr);
         }
     }
 

lib/extension/onEvent.ts

@@ -43,7 +43,7 @@ export default class OnEvent extends Extension {
         zhc.onEvent(type, data, device.zh);
 
         if (device.definition?.onEvent) {
-            await device.definition.onEvent(type, data, device.zh, device.settings);
+            await device.definition.onEvent(type, data, device.zh, device.options);
         }
     }
 }

lib/extension/otaUpdate.ts

@@ -3,9 +3,24 @@ import logger from '../util/logger';
 import stringify from 'json-stable-stringify-without-jsonify';
 import utils from '../util/utils';
 import tradfriOTA from 'zigbee-herdsman-converters/lib/ota/tradfri';
+import zigbeeOTA from 'zigbee-herdsman-converters/lib/ota/zigbeeOTA';
 import Extension from './extension';
 import bind from 'bind-decorator';
 import Device from '../model/device';
+import dataDir from '../util/data';
+import * as URI from 'uri-js';
+import path from 'path';
+
+function isValidUrl(url: string): boolean {
+    let parsed;
+    try {
+        parsed = URI.parse(url);
+    } catch (_) {
+        // istanbul ignore next
+        return false;
+    }
+    return parsed.scheme === 'http' || parsed.scheme === 'https';
+}
 
 type UpdateState = 'updating' | 'idle' | 'available';
 interface UpdatePayload {
@@ -24,13 +39,27 @@ export default class OTAUpdate extends Extension {
     override async start(): Promise<void> {
         this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
         this.eventBus.onDeviceMessage(this, this.onZigbeeEvent);
-        if (settings.get().advanced.ikea_ota_use_test_url) {
+        if (settings.get().ota.ikea_ota_use_test_url) {
             tradfriOTA.useTestURL();
         }
 
+        // Let zigbeeOTA module know if the override index file is provided
+        let overrideOTAIndex = settings.get().ota.zigbee_ota_override_index_location;
+        if (overrideOTAIndex) {
+            // If the file name is not a full path, then treat it as a relative to the data directory
+            if (!isValidUrl(overrideOTAIndex) && !path.isAbsolute(overrideOTAIndex)) {
+                overrideOTAIndex = dataDir.joinPath(overrideOTAIndex);
+            }
+
+            zigbeeOTA.useIndexOverride(overrideOTAIndex);
+        }
+
+        // In order to support local firmware files we need to let zigbeeOTA know where the data directory is
+        zigbeeOTA.setDataDir(dataDir.getPath());
+
+        // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
+        // remove them.
         for (const device of this.zigbee.devices(false)) {
-            // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
-            // remove them.
             this.removeProgressAndRemainingFromState(device);
         }
     }
@@ -92,11 +121,11 @@ export default class OTAUpdate extends Extension {
         }
     }
 
-    private async readSoftwareBuildIDAndDateCode(device: Device, sendWhenActive: boolean):
+    private async readSoftwareBuildIDAndDateCode(device: Device, sendWhen: 'active' | 'immediate'):
         Promise<{softwareBuildID: string, dateCode: string}> {
         try {
             const endpoint = device.zh.endpoints.find((e) => e.supportsInputCluster('genBasic'));
-            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId'], {sendWhenActive});
+            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId'], {sendWhen});
             return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
         } catch (e) {
             return null;
@@ -223,17 +252,16 @@ export default class OTAUpdate extends Extension {
                         }
                     };
 
-                    const from_ = await this.readSoftwareBuildIDAndDateCode(device, false);
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(device, 'immediate');
                     await device.definition.ota.updateToLatest(device.zh, logger, onProgress);
-                    const to = await this.readSoftwareBuildIDAndDateCode(device, device.zh.type === 'EndDevice');
-                    const [fromS, toS] = [stringify(from_), stringify(to)];
+                    logger.info(`Finished update of '${device.name}'`);
                     this.eventBus.emitReconfigure({device});
-                    const msg = `Finished update of '${device.name}'` +
-                        (to ? `, from '${fromS}' to '${toS}'` : ``);
-                    logger.info(msg);
                     this.removeProgressAndRemainingFromState(device);
                     const payload = this.getEntityPublishPayload('idle');
                     this.publishEntityState(device, payload);
+                    const to = await this.readSoftwareBuildIDAndDateCode(device, 'active');
+                    const [fromS, toS] = [stringify(from_), stringify(to)];
+                    logger.info(`Device '${device.name}' was updated from '${fromS}' to '${toS}'`);
                     responseData.from = from_ ? utils.toSnakeCase(from_) : null;
                     responseData.to = to ? utils.toSnakeCase(to) : null;
                     this.eventBus.emitDevicesChanged();

lib/extension/publish.ts

@@ -10,7 +10,7 @@ import Device from '../model/device';
 import bind from 'bind-decorator';
 
 const topicRegex = new RegExp(`^(.+?)(?:/(${utils.endpointNames.join('|')}|\\d+))?/(get|set)(?:/(.+))?`);
-const propertyEndpointRegex = new RegExp(`^(.*)_(${utils.endpointNames.join('|')})$`);
+const propertyEndpointRegex = new RegExp(`^(.*?)_(${utils.endpointNames.join('|')})$`);
 const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
 const sceneConverterKeys = ['scene_store', 'scene_add', 'scene_remove', 'scene_remove_all'];
 
@@ -91,7 +91,7 @@ export default class Publish extends Extension {
         // Only do this when the retrieve_state option is enabled for this device.
         // retrieve_state == decprecated
         if (re instanceof Device && result && result.hasOwnProperty('readAfterWriteTime') &&
-            re.settings.retrieve_state
+            re.options.retrieve_state
         ) {
             setTimeout(() => converter.convertGet(target, key, meta), result.readAfterWriteTime);
         }
@@ -134,7 +134,7 @@ export default class Publish extends Extension {
             return;
         }
         const device = re instanceof Device ? re.zh : null;
-        const entitySettings = re.settings;
+        const entitySettings = re.options;
         const entityState = this.state.get(re) || {};
         const membersState = re instanceof Group ?
             Object.fromEntries(re.zh.members.map((e) => [e.getDevice().ieeeAddr,

lib/extension/receive.ts

@@ -91,23 +91,6 @@ export default class Receive extends Extension {
         /* istanbul ignore next */
         if (!data.device) return;
 
-        /**
-         * Handling of re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The network address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the network address of the end device.
-         */
-        if (data.device.isXiaomi() && data.device.zh.type === 'Router' && data.groupID) {
-            logger.debug('Handling re-transmitted Xiaomi message');
-            data = {...data, device: this.zigbee.deviceByNetworkAddress(data.groupID)};
-        }
-
         if (!this.shouldProcess(data)) {
             utils.publishLastSeen({device: data.device, reason: 'messageEmitted'},
                 settings.get(), true, this.publishEntityState);
@@ -145,9 +128,9 @@ export default class Receive extends Extension {
             }
 
             // Check if we have to debounce
-            if (data.device.settings.debounce) {
-                this.publishDebounce(data.device, payload, data.device.settings.debounce,
-                    data.device.settings.debounce_ignore);
+            if (data.device.options.debounce) {
+                this.publishDebounce(data.device, payload, data.device.options.debounce,
+                    data.device.options.debounce_ignore);
             } else {
                 this.publishEntityState(data.device, payload);
             }
@@ -158,7 +141,7 @@ export default class Receive extends Extension {
         for (const converter of converters) {
             try {
                 const converted = await converter.convert(
-                    data.device.definition, data, publish, data.device.settings, meta);
+                    data.device.definition, data, publish, data.device.options, meta);
                 if (converted) {
                     payload = {...payload, ...converted};
                 }

lib/model/device.ts

@@ -5,16 +5,20 @@ import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
 export default class Device {
     public zh: zh.Device;
     private _definition: zhc.Definition;
+    private _definitionModelID: string;
 
     get ieeeAddr(): string {return this.zh.ieeeAddr;}
     get ID(): string {return this.zh.ieeeAddr;}
-    get settings(): DeviceSettings {return {...settings.get().device_options, ...settings.getDevice(this.ieeeAddr)};}
+    get options(): DeviceOptions {return {...settings.get().device_options, ...settings.getDevice(this.ieeeAddr)};}
     get name(): string {
-        return this.zh.type === 'Coordinator' ? 'Coordinator' : this.settings?.friendly_name || this.ieeeAddr;
+        return this.zh.type === 'Coordinator' ? 'Coordinator' : this.options?.friendly_name || this.ieeeAddr;
     }
     get definition(): zhc.Definition {
-        if (!this._definition && !this.zh.interviewing) {
+        // Some devices can change modelID, reconsider the definition in that case.
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/3016
+        if (!this.zh.interviewing && (!this._definition || this._definitionModelID !== this.zh.modelID)) {
             this._definition = zigbeeHerdsmanConverters.findByDevice(this.zh);
+            this._definitionModelID = this.zh.modelID;
         }
         return this._definition;
     }
@@ -26,7 +30,7 @@ export default class Device {
     exposes(): zhc.DefinitionExpose[] {
         /* istanbul ignore if */
         if (typeof this.definition.exposes == 'function') {
-            return this.definition.exposes(this.zh, this.settings);
+            return this.definition.exposes(this.zh, this.options);
         } else {
             return this.definition.exposes;
         }
@@ -67,13 +71,6 @@ export default class Device {
         return name === 'default' ? null : name;
     }
 
-    isXiaomi(): boolean {
-        const xiaomiManufacturerID = [4151, 4447];
-        /* istanbul ignore next */
-        return this.zh.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(this.zh.manufacturerID) &&
-            (!this.zh.manufacturerName || !this.zh.manufacturerName.startsWith('Trust'));
-    }
-
     isIkeaTradfri(): boolean {return this.zh.manufacturerID === 4476;}
 
     isDevice(): this is Device {return true;}

lib/model/group.ts

@@ -4,13 +4,23 @@ import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
 
 export default class Group {
     public zh: zh.Group;
+    private resolveDevice: (ieeeAddr: string) => Device;
 
     get ID(): number {return this.zh.groupID;}
-    get settings(): GroupSettings {return settings.getGroup(this.ID);}
-    get name(): string {return this.settings?.friendly_name || this.ID.toString();}
+    get options(): GroupOptions {return {...settings.getGroup(this.ID)};}
+    get name(): string {return this.options?.friendly_name || this.ID.toString();}
 
-    constructor(group: zh.Group) {
+    constructor(group: zh.Group, resolveDevice: (ieeeAddr: string) => Device) {
         this.zh = group;
+        this.resolveDevice = resolveDevice;
+    }
+
+    hasMember(device: Device): boolean {
+        return !!device.zh.endpoints.find((e) => this.zh.members.includes(e));
+    }
+
+    membersDevices(): Device[] {
+        return this.zh.members.map((e) => this.resolveDevice(e.getDevice().ieeeAddr)).filter((d) => d);
     }
 
     membersDefinitions(): zhc.Definition[] {

lib/mqtt.ts

@@ -22,7 +22,7 @@ export default class MQTT {
         const options: mqtt.IClientOptions = {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
-                payload: 'offline',
+                payload: utils.availabilityPayload('offline', settings.get()),
                 retain: settings.get().mqtt.force_disable_retain ? false : true,
                 qos: 1,
             },
@@ -71,6 +71,7 @@ export default class MQTT {
             this.client = mqtt.connect(mqttSettings.server, options);
             // @ts-ignore https://github.com/Koenkk/zigbee2mqtt/issues/9822
             this.client.stream.setMaxListeners(0);
+            this.eventBus.onPublishAvailability(this, () => this.publishStateOnline());
 
             const onConnect = this.onConnect;
             this.client.on('connect', async () => {
@@ -94,12 +95,18 @@ export default class MQTT {
 
         logger.info('Connected to MQTT server');
         this.subscribe(`${settings.get().mqtt.base_topic}/#`);
-        await this.publish('bridge/state', 'online', {retain: true, qos: 0});
+        await this.publishStateOnline();
+    }
+
+    async publishStateOnline(): Promise<void> {
+        await this.publish('bridge/state', utils.availabilityPayload('online', settings.get()), {retain: true, qos: 0});
     }
 
     async disconnect(): Promise<void> {
         clearTimeout(this.connectionTimer);
-        await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
+        await this.publish('bridge/state', utils.availabilityPayload('offline', settings.get()),
+            {retain: true, qos: 0});
+        this.eventBus.removeListeners(this);
         logger.info('Disconnecting from MQTT server');
         this.client.end();
     }

lib/types/types.d.ts

@@ -115,7 +115,8 @@ declare global {
             options: zhc.DefinitionExpose[],
             vendor: string
             exposes: DefinitionExpose[] | ((device: zh.Device, options: KeyValue) => DefinitionExpose[])
-            configure?: (device: zh.Device, coordinatorEndpoint: zh.Endpoint, logger: Logger) => Promise<void>;
+            configure?: (device: zh.Device, coordinatorEndpoint: zh.Endpoint, logger: Logger,
+                options?: DeviceOptions) => Promise<void>;
             onEvent?: (type: string, data: KeyValue, device: zh.Device, settings: KeyValue) => Promise<void>;
             ota?: {
                 isUpdateAvailable: (device: zh.Device, logger: Logger, data?: KeyValue) => Promise<boolean>;
@@ -129,7 +130,7 @@ declare global {
     }
 
     namespace eventdata {
-        type DeviceRenamed = { device: Device, homeAssisantRename: boolean, from: string, to: string };
+        type EntityRenamed = { entity: Device | Group, homeAssisantRename: boolean, from: string, to: string };
         type DeviceRemoved = { ieeeAddr: string, name: string };
         type MQTTMessage = { topic: string, message: string };
         type MQTTMessagePublished = { topic: string, payload: string, options: {retain: boolean, qos: number} };
@@ -164,24 +165,18 @@ declare global {
     // Settings
     // eslint-disable camelcase
     interface Settings {
-        homeassistant?: boolean,
-        devices?: {[s: string]: DeviceSettings},
-        groups?: {[s: string]: GroupSettings},
-        passlist: string[],
-        blocklist: string[],
-        whitelist: string[],
-        ban: string[],
-        availability?: boolean | {
-            active?: {timeout?: number},
-            passive?: {timeout?: number}
+        homeassistant?: {
+            discovery_topic: string,
+            status_topic: string,
+            legacy_entity_attributes: boolean,
+            legacy_triggers: boolean,
         },
-        permit_join: boolean,
-        frontend?: {
-            auth_token?: string,
-            host?: string,
-            port?: number,
-            url?: string,
+        permit_join?: boolean,
+        availability?: {
+            active: {timeout: number},
+            passive: {timeout: number}
         },
+        external_converters: string[],
         mqtt: {
             base_topic: string,
             include_device_information: boolean,
@@ -198,11 +193,14 @@ declare global {
             reject_unauthorized?: boolean,
         },
         serial: {
-            disable_led?: boolean,
+            disable_led: boolean,
             port?: string,
-            adapter?: 'deconz' | 'zstack' | 'ezsp' | 'zigate'
+            adapter?: 'deconz' | 'zstack' | 'ezsp' | 'zigate',
+            baudrate?: number,
+            rtscts?: boolean,
         },
-        device_options: KeyValue,
+        passlist: string[],
+        blocklist: string[],
         map_options: {
             graphviz: {
                 colors: {
@@ -223,12 +221,24 @@ declare global {
                 },
             },
         },
-        experimental: {
-            output: 'json' | 'attribute' | 'attribute_and_json',
-            transmit_power?: number,
+        ota: {
+            update_check_interval: number,
+            disable_automatic_update_check: boolean,
+            zigbee_ota_override_index_location?: string,
+            ikea_ota_use_test_url?: boolean,
         },
+        frontend?: {
+            auth_token?: string,
+            host?: string,
+            port?: number,
+            url?: string,
+        },
+        devices?: {[s: string]: DeviceOptions},
+        groups?: {[s: string]: GroupOptions},
+        device_options: KeyValue,
         advanced: {
             legacy_api: boolean,
+            legacy_availability_payload: boolean,
             log_rotation: boolean,
             log_symlink_current: boolean,
             log_output: ('console' | 'file' | 'syslog')[],
@@ -236,7 +246,6 @@ declare global {
             log_file: string,
             log_level: 'debug' | 'info' | 'error' | 'warn',
             log_syslog: KeyValue,
-            soft_reset_timeout: number,
             pan_id: number | 'GENERATE',
             ext_pan_id: number[],
             channel: number,
@@ -248,30 +257,21 @@ declare global {
             last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' | 'epoch',
             elapsed: boolean,
             network_key: number[] | 'GENERATE',
-            report: boolean,
-            homeassistant_discovery_topic: string,
-            homeassistant_status_topic: string,
-            homeassistant_legacy_entity_attributes: boolean,
-            homeassistant_legacy_triggers: boolean,
             timestamp_format: string,
-            baudrate?: number,
-            rtscts?: boolean,
-            ikea_ota_use_test_url?: boolean,
-            // below are deprecated
+            output: 'json' | 'attribute' | 'attribute_and_json',
+            transmit_power?: number,
+            // Everything below is deprecated
             availability_timeout?: number,
             availability_blocklist?: string[],
             availability_passlist?: string[],
             availability_blacklist?: string[],
             availability_whitelist?: string[],
+            soft_reset_timeout: number,
+            report: boolean,
         },
-        ota: {
-            update_check_interval: number,
-            disable_automatic_update_check: boolean
-        },
-        external_converters: string[],
     }
 
-    interface DeviceSettings {
+    interface DeviceOptions {
         ID?: string,
         retention?: number,
         availability?: boolean | {timeout: number},
@@ -285,10 +285,11 @@ declare global {
         legacy?: boolean,
         filtered_attributes?: string[],
         friendly_name: string,
+        description?: string,
         qos?: 0 | 1 | 2,
     }
 
-    interface GroupSettings {
+    interface GroupOptions {
         devices?: string[],
         ID?: number,
         optimistic?: boolean,
@@ -297,6 +298,7 @@ declare global {
         homeassistant?: KeyValue,
         filtered_attributes?: string[],
         friendly_name: string,
+        description?: string,
         qos?: 0 | 1 | 2,
     }
 }

lib/types/zigbee-herdsman-converters.d.ts

@@ -11,3 +11,7 @@ declare module 'zigbee-herdsman-converters/lib/ota/tradfri' {
     export function useTestURL(): void;
 }
 
+declare module 'zigbee-herdsman-converters/lib/ota/zigbeeOTA' {
+    export function useIndexOverride(indexFileName: string): void;
+    export function setDataDir(dataDir: string): void;
+}

lib/util/logger.ts

@@ -62,7 +62,12 @@ const transportsToUse: winston.transport[] = [
             winston.format.printf(/* istanbul ignore next */(info) => {
                 const {timestamp, level, message} = info;
                 const l = winstonToZ2mLevel(level as WinstonLogLevel);
-                const prefix = colorizer.colorize(l, `Zigbee2MQTT:${levelWithCompensatedLength[l]}`);
+
+                const plainPrefix = `Zigbee2MQTT:${levelWithCompensatedLength[l]}`;
+                let prefix = plainPrefix;
+                if (process.stdout.isTTY) {
+                    prefix = colorizer.colorize(l, plainPrefix);
+                }
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
         ),
@@ -91,8 +96,10 @@ if (settings.get().advanced.log_rotation) {
     transportFileOptions.maxsize = 10000000; // 10MB
 }
 
+let fileTransport : winston.transport;
 if (output.includes('file')) {
-    transportsToUse.push(new winston.transports.File(transportFileOptions));
+    fileTransport = new winston.transports.File(transportFileOptions);
+    transportsToUse.push(fileTransport);
 }
 
 /* istanbul ignore next */
@@ -181,6 +188,28 @@ function error(message: string): void {
     logger.error(message);
 }
 
+// Workaround for https://github.com/winstonjs/winston/issues/1629.
+// https://github.com/Koenkk/zigbee2mqtt/pull/10905
+/* istanbul ignore next */
+async function end(): Promise<void> {
+    logger.end();
+
+    await new Promise<void>((resolve) => {
+        if (!fileTransport) {
+            process.nextTick(resolve);
+        } else {
+            // @ts-ignore
+            if (fileTransport._dest) {
+                // @ts-ignore
+                fileTransport._dest.on('finish', resolve);
+            } else {
+                // @ts-ignore
+                fileTransport.on('open', () => fileTransport._dest.on('finish', resolve));
+            }
+        }
+    });
+}
+
 export default {
-    logOutput, warn, warning, error, info, debug, setLevel, getLevel, cleanup, addTransport, winston: logger,
+    logOutput, warn, warning, error, info, debug, setLevel, getLevel, cleanup, addTransport, end, winston: logger,
 };

lib/util/settings.schema.json

@@ -1,14 +1,49 @@
 {
   "type": "object",
   "properties": {
-    "device_options": {
-      "type": "object"
-    },
     "homeassistant": {
       "title": "Home Assistant integration",
-      "type": "boolean",
+      "requiresRestart": true,
       "description": "Home Assistant integration (MQTT discovery)",
-      "default": false
+      "default": false,
+      "oneOf": [
+        {
+          "type": "boolean",
+          "title": "Home Assistant (simple)"
+        },
+        {
+          "type": "object",
+          "title": "Home Assistant (advanced)",
+          "properties": {
+            "legacy_triggers": {
+              "type": "boolean",
+              "title": "Home Assistant legacy triggers",
+              "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
+              "default": true
+            },
+            "discovery_topic": {
+              "type": "string",
+              "title": "Homeassistant discovery topic",
+              "description": "Home Assistant discovery topic",
+              "requiresRestart": true,
+              "examples": ["homeassistant"]
+            },
+            "legacy_entity_attributes": {
+              "type": "boolean",
+              "title": "Home Assistant legacy entity attributes",
+              "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+              "default": true
+            },
+            "status_topic": {
+              "type": "string",
+              "title": "Home Assistant status topic",
+              "description": "Home Assistant status topic",
+              "requiresRestart": true,
+              "examples": ["homeassistant/status"]
+            }
+          }
+        }
+      ]
     },
     "permit_join": {
       "type": "boolean",
@@ -16,25 +51,15 @@
       "title": "Permit join",
       "description": "Allow new devices to join (re-applied at restart)"
     },
-    "external_converters": {
-      "type": "array",
-      "title": "External converters",
-      "description": "You can define external converters to e.g. add support for a DiY device",
-      "requiresRestart": true,
-      "items": {
-        "type": "string"
-      },
-      "examples": ["DIYRuZ_FreePad.js"]
-    },
     "availability": {
       "oneOf": [
         {
           "type": "boolean",
-          "title": "Availability (boolean)"
+          "title": "Availability (simple)"
         },
         {
           "type": "object",
-          "title": "Availability (object)",
+          "title": "Availability (advanced)",
           "properties": {
             "active": {
               "type": "object",
@@ -73,6 +98,16 @@
       "requiresRestart": true,
       "description": "Checks whether devices are online/offline"
     },
+    "external_converters": {
+      "type": "array",
+      "title": "External converters",
+      "description": "You can define external converters to e.g. add support for a DiY device",
+      "requiresRestart": true,
+      "items": {
+        "type": "string"
+      },
+      "examples": ["DIYRuZ_FreePad.js"]
+    },
     "mqtt": {
       "type": "object",
       "title": "MQTT",
@@ -80,6 +115,7 @@
         "base_topic": {
           "type": "string",
           "title": "Base topic",
+          "default": "zigbee2mqtt",
           "requiresRestart": true,
           "description": "MQTT base topic for Zigbee2MQTT MQTT messages",
           "examples": ["zigbee2mqtt"]
@@ -169,7 +205,7 @@
           "default": false
         }
       },
-      "required": ["base_topic", "server"]
+      "required": ["server"]
     },
     "serial": {
       "type": "object",
@@ -196,6 +232,19 @@
           "default": "auto",
           "requiresRestart": true,
           "description": "Adapter type, not needed unless you are experiencing problems"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
         }
       }
     },
@@ -217,331 +266,6 @@
         "type": "string"
       }
     },
-    "whitelist": {
-      "readOnly": true,
-      "type": "array",
-      "requiresRestart": true,
-      "title": "Whitelist (deprecated, use passlist)",
-      "items": {
-        "type": "string"
-      }
-    },
-    "ban": {
-      "readOnly": true,
-      "type": "array",
-      "requiresRestart": true,
-      "title": "Ban (deprecated, use blocklist)",
-      "items": {
-        "type": "string"
-      }
-    },
-    "experimental": {
-      "type": "object",
-      "title": "Experimental",
-      "properties": {
-        "transmit_power": {
-          "type": ["number", "null"],
-          "title": "Transmit power",
-          "requiresRestart": true,
-          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
-        },
-        "output": {
-          "type": "string",
-          "enum": ["attribute_and_json", "attribute", "json"],
-          "title": "MQTT output type",
-          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
-        }
-      }
-    },
-    "advanced": {
-      "type": "object",
-      "title": "Advanced",
-      "properties": {
-        "legacy_api": {
-          "type": "boolean",
-          "title": "Legacy API",
-          "requiresRestart": true,
-          "description": "Disables the legacy api (false = disable)",
-          "default": true
-        },
-        "pan_id": {
-          "oneOf": [{
-              "type": "string",
-              "title": "Pan ID (string)"
-            },
-            {
-              "type": "number",
-              "title": "Pan ID (number)"
-            }
-          ],
-          "title": "Pan ID",
-          "requiresRestart": true,
-          "description": "ZigBee pan ID, changing requires repairing all devices!"
-        },
-        "ext_pan_id": {
-          "type": "array",
-          "items": {
-            "type": "number"
-          },
-          "title": "Ext Pan ID",
-          "requiresRestart": true,
-          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
-        },
-        "channel": {
-          "type": "number",
-          "minimum": 11,
-          "maximum": 26,
-          "default": 11,
-          "title": "ZigBee channel",
-          "requiresRestart": true,
-          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
-          "examples": [15, 20, 25]
-        },
-        "cache_state": {
-          "type": "boolean",
-          "title": "Cache state",
-          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
-          "default": true
-        },
-        "cache_state_persistent": {
-          "type": "boolean",
-          "title": "Persist cache state",
-          "description": "Persist cached state, only used when cache_state: true",
-          "default": true
-        },
-        "cache_state_send_on_startup": {
-          "type": "boolean",
-          "title": "Send cached state on startup",
-          "description": "Send cached state on startup, only used when cache_state: true",
-          "default": true
-        },
-        "log_rotation": {
-          "type": "boolean",
-          "title": "Log rotation",
-          "requiresRestart": true,
-          "description": "Log rotation",
-          "default": true
-        },
-        "log_symlink_current": {
-          "type": "boolean",
-          "title": "Log symlink current",
-          "requiresRestart": true,
-          "description": "Create symlink to current logs in the log directory",
-          "default": false
-        },
-        "log_level": {
-          "type": "string",
-          "enum": ["info", "warn", "error", "debug"],
-          "title": "Log level",
-          "description": "Logging level",
-          "default": "info"
-        },
-        "log_output": {
-          "type": "array",
-          "requiresRestart": true,
-          "items": {
-            "type": "string",
-            "enum": ["console", "file", "syslog"]
-          },
-          "title": "Log output",
-          "description": "Output location of the log, leave empty to supress logging"
-        },
-        "log_directory": {
-          "type": "string",
-          "title": "Log directory",
-          "requiresRestart": true,
-          "description": "Location of log directory",
-          "examples": ["data/log/%TIMESTAMP%"]
-        },
-        "log_file": {
-          "type": "string",
-          "title": "Log file",
-          "requiresRestart": true,
-          "description": "Log file name, can also contain timestamp",
-          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
-          "default": "log.txt"
-        },
-        "baudrate": {
-          "type": "number",
-          "title": "Baudrate",
-          "requiresRestart": true,
-          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
-          "examples": [38400, 57600, 115200]
-        },
-        "rtscts": {
-          "type": "boolean",
-          "title": "RTS / CTS",
-          "requiresRestart": true,
-          "description": "RTS / CTS Hardware Flow Control for serial port"
-        },
-        "soft_reset_timeout": {
-          "type": "number",
-          "minimum": 0,
-          "requiresRestart": true,
-          "title": "Soft reset timeout (deprecated)",
-          "description": "Soft reset ZNP after timeout",
-          "readOnly": true
-        },
-        "network_key": {
-          "oneOf": [{
-              "type": "string",
-              "title": "Network key(string)"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "number"
-              },
-              "title": "Network key(array)"
-            }
-          ],
-          "title": "Network key",
-          "requiresRestart": true,
-          "description": "Network encryption key, changing requires repairing all devices!"
-        },
-        "last_seen": {
-          "type": "string",
-          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
-          "title": "Last seen",
-          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
-          "default": "disable"
-        },
-        "elapsed": {
-          "type": "boolean",
-          "title": "Elapsed",
-          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
-          "default": false
-        },
-        "report": {
-          "type": "boolean",
-          "title": "Reporting",
-          "requiresRestart": true,
-          "readOnly": true,
-          "description": "Enables report feature (deprecated)"
-        },
-        "homeassistant_discovery_topic": {
-          "type": "string",
-          "title": "Homeassistant discovery topic",
-          "description": "Home Assistant discovery topic",
-          "requiresRestart": true,
-          "examples": ["homeassistant"]
-        },
-        "homeassistant_legacy_entity_attributes": {
-          "type": "boolean",
-          "title": "Home Assistant legacy entity attributes",
-          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
-          "default": true
-        },
-        "homeassistant_status_topic": {
-          "type": "string",
-          "title": "Home Assistant status topic",
-          "description": "Home Assistant status topic",
-          "requiresRestart": true,
-          "examples": ["homeassistant/status"]
-        },
-        "timestamp_format": {
-          "type": "string",
-          "title": "Timestamp format",
-          "requiresRestart": true,
-          "description": "Log timestamp format",
-          "examples": ["YYYY-MM-DD HH:mm:ss"]
-        },
-        "adapter_concurrent": {
-          "title": "Adapter concurrency",
-          "requiresRestart": true,
-          "type": ["number", "null"],
-          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
-        },
-        "adapter_delay": {
-          "type": ["number", "null"],
-          "requiresRestart": true,
-          "title": "Adapter delay",
-          "description": "Adapter delay"
-        },
-        "ikea_ota_use_test_url": {
-          "type": "boolean",
-          "title": "IKEA TRADFRI OTA use test url",
-          "requiresRestart": true,
-          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
-          "default": false
-        },
-        "homeassistant_legacy_triggers": {
-          "type": "boolean",
-          "title": "Home Assistant legacy triggers",
-          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
-          "default": true
-        },
-        "log_syslog": {
-          "type": "object",
-          "title": "syslog",
-          "properties": {
-            "host": {
-              "type": "string",
-              "title": "Host",
-              "description": "The host running syslogd, defaults to localhost.",
-              "default": "localhost"
-            },
-            "port": {
-              "type": "number",
-              "title": "Port",
-              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
-              "default": 123
-            },
-            "protocol": {
-              "type": "string",
-              "title": "Protocol",
-              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
-              "default": "tcp4",
-              "examples": [
-                "udp4",
-                "tls4",
-                "unix",
-                "unix-connect"
-              ]
-            },
-            "path": {
-              "type": "string",
-              "title": "Path",
-              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
-              "default": "/dev/log",
-              "examples": [
-                "/var/run/syslog"
-              ]
-            },
-            "pid": {
-              "type": "string",
-              "title": "PID",
-              "description": "PID of the process that log messages are coming from (Default process.pid).",
-              "default": "process.pid"
-            },
-            "localhost": {
-              "type": "string",
-              "title": "Localhost",
-              "description": "Host to indicate that log messages are coming from (Default: localhost).",
-              "default": "localhost"
-            },
-            "type": {
-              "type": "string",
-              "title": "Type",
-              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
-              "default": "5424"
-            },
-            "app_name": {
-              "type": "string",
-              "title": "Localhost",
-              "description": "The name of the application (Default: Zigbee2MQTT).",
-              "default": "Zigbee2MQTT"
-            },
-            "eol": {
-              "type": "string",
-              "title": "eol",
-              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
-              "default": "/n"
-            }
-          }
-        }
-      }
-    },
     "map_options": {
       "type": "object",
       "title": "Networkmap",
@@ -612,9 +336,65 @@
           "title": "Disable automatic update check",
           "description": "Zigbee devices may request a firmware update, and do so frequently, causing Zigbee2MQTT to reach out to third party servers. If you disable these device initiated checks, you can still initiate a firmware update check manually.",
           "default": false
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
+        },
+        "zigbee_ota_override_index_location": {
+          "type": "string",
+          "title": "OTA index override file name",
+          "requiresRestart": true,
+          "description": "Location of override OTA index file",
+          "examples": ["index.json"]
         }
       }
     },
+    "frontend": {
+      "oneOf": [
+        {
+          "type": "boolean",
+          "title": "Frontend (simple)"
+        },
+        {
+          "type": "object",
+          "title": "Frontend (advanced)",
+          "properties": {
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "Frontend binding port",
+              "default": 8080,
+              "requiresRestart": true
+            },
+            "host": {
+              "type": "string",
+              "title": "Bind host",
+              "description": "Frontend binding host",
+              "default": "0.0.0.0",
+              "requiresRestart": true
+            },
+            "auth_token": {
+              "type": ["string", "null"],
+              "title": "Auth token",
+              "description": "Enables authentication, disabled by default",
+              "requiresRestart": true
+            },
+            "url": {
+              "type": ["string", "null"],
+              "title": "URL",
+              "description": "URL on which the frontend can be reached, currently only used for the Home Assistant device configuration page",
+              "requiresRestart": true
+            }
+          }
+        }
+      ],
+      "title": "Frontend",
+      "requiresRestart": true
+    },
     "devices": {
       "type": "object",
       "propertyNames": {
@@ -637,31 +417,353 @@
         }
       }
     },
-    "frontend": {
+    "device_options": {
       "type": "object",
-      "title": "Frontend",
+      "title": "Options that are applied to all devices"
+    },
+    "advanced": {
+      "type": "object",
+      "title": "Advanced",
       "properties": {
-        "port": {
-          "type": "number",
-          "title": "Port",
-          "description": "Frontend binding port",
-          "default": 8080,
-          "requiresRestart": true
+        "legacy_api": {
+          "type": "boolean",
+          "title": "Legacy API",
+          "requiresRestart": true,
+          "description": "Disables the legacy api (false = disable)",
+          "default": true
         },
-        "host": {
+        "legacy_availability_payload": {
+          "type": "boolean",
+          "title": "Legacy availability payload",
+          "requiresRestart": true,
+          "description": "Payload to be used for device availabilty and bridge/state topics. true = text, false = JSON",
+          "default": true
+        },
+        "log_rotation": {
+          "type": "boolean",
+          "title": "Log rotation",
+          "requiresRestart": true,
+          "description": "Log rotation",
+          "default": true
+        },
+        "log_symlink_current": {
+          "type": "boolean",
+          "title": "Log symlink current",
+          "requiresRestart": true,
+          "description": "Create symlink to current logs in the log directory",
+          "default": false
+        },
+        "log_output": {
+          "type": "array",
+          "requiresRestart": true,
+          "items": {
+            "type": "string",
+            "enum": ["console", "file", "syslog"]
+          },
+          "title": "Log output",
+          "description": "Output location of the log, leave empty to suppress logging"
+        },
+        "log_directory": {
           "type": "string",
-          "title": "Bind host",
-          "description": "Frontend binding host",
-          "default": "0.0.0.0",
-          "requiresRestart": true
+          "title": "Log directory",
+          "requiresRestart": true,
+          "description": "Location of log directory",
+          "examples": ["data/log/%TIMESTAMP%"]
         },
-        "auth_token": {
-          "type": ["string", "null"],
-          "title": "Auth token",
-          "description": "Enables authentication, disabled by default",
-          "requiresRestart": true
+        "log_file": {
+          "type": "string",
+          "title": "Log file",
+          "requiresRestart": true,
+          "description": "Log file name, can also contain timestamp",
+          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
+          "default": "log.txt"
+        },
+        "log_level": {
+          "type": "string",
+          "enum": ["info", "warn", "error", "debug"],
+          "title": "Log level",
+          "description": "Logging level",
+          "default": "info"
+        },
+        "log_syslog": {
+          "type": "object",
+          "title": "syslog",
+          "properties": {
+            "host": {
+              "type": "string",
+              "title": "Host",
+              "description": "The host running syslogd, defaults to localhost.",
+              "default": "localhost"
+            },
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
+              "default": 123
+            },
+            "protocol": {
+              "type": "string",
+              "title": "Protocol",
+              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
+              "default": "tcp4",
+              "examples": [
+                "udp4",
+                "tls4",
+                "unix",
+                "unix-connect"
+              ]
+            },
+            "path": {
+              "type": "string",
+              "title": "Path",
+              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
+              "default": "/dev/log",
+              "examples": [
+                "/var/run/syslog"
+              ]
+            },
+            "pid": {
+              "type": "string",
+              "title": "PID",
+              "description": "PID of the process that log messages are coming from (Default process.pid).",
+              "default": "process.pid"
+            },
+            "localhost": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "Host to indicate that log messages are coming from (Default: localhost).",
+              "default": "localhost"
+            },
+            "type": {
+              "type": "string",
+              "title": "Type",
+              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
+              "default": "5424"
+            },
+            "app_name": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "The name of the application (Default: Zigbee2MQTT).",
+              "default": "Zigbee2MQTT"
+            },
+            "eol": {
+              "type": "string",
+              "title": "eol",
+              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
+              "default": "/n"
+            }
+          }
+        },
+        "pan_id": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Pan ID (string)"
+            },
+            {
+              "type": "number",
+              "title": "Pan ID (number)"
+            }
+          ],
+          "title": "Pan ID",
+          "requiresRestart": true,
+          "description": "ZigBee pan ID, changing requires repairing all devices!"
+        },
+        "ext_pan_id": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "title": "Ext Pan ID",
+          "requiresRestart": true,
+          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
+        },
+        "channel": {
+          "type": "number",
+          "minimum": 11,
+          "maximum": 26,
+          "default": 11,
+          "title": "ZigBee channel",
+          "requiresRestart": true,
+          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
+          "examples": [15, 20, 25]
+        },
+        "adapter_concurrent": {
+          "title": "Adapter concurrency",
+          "requiresRestart": true,
+          "type": ["number", "null"],
+          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
+        },
+        "adapter_delay": {
+          "type": ["number", "null"],
+          "requiresRestart": true,
+          "title": "Adapter delay",
+          "description": "Adapter delay"
+        },
+        "cache_state": {
+          "type": "boolean",
+          "title": "Cache state",
+          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
+          "default": true
+        },
+        "cache_state_persistent": {
+          "type": "boolean",
+          "title": "Persist cache state",
+          "description": "Persist cached state, only used when cache_state: true",
+          "default": true
+        },
+        "cache_state_send_on_startup": {
+          "type": "boolean",
+          "title": "Send cached state on startup",
+          "description": "Send cached state on startup, only used when cache_state: true",
+          "default": true
+        },
+        "last_seen": {
+          "type": "string",
+          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
+          "title": "Last seen",
+          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
+          "default": "disable"
+        },
+        "elapsed": {
+          "type": "boolean",
+          "title": "Elapsed",
+          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
+          "default": false
+        },
+        "network_key": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Network key(string)"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "number"
+              },
+              "title": "Network key(array)"
+            }
+          ],
+          "title": "Network key",
+          "requiresRestart": true,
+          "description": "Network encryption key, changing requires repairing all devices!"
+        },
+        "timestamp_format": {
+          "type": "string",
+          "title": "Timestamp format",
+          "requiresRestart": true,
+          "description": "Log timestamp format",
+          "examples": ["YYYY-MM-DD HH:mm:ss"]
+        },
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        },
+        "homeassistant_discovery_topic": {
+          "type": "string",
+          "title": "Homeassistant discovery topic",
+          "description": "Home Assistant discovery topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant"]
+        },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
+        "homeassistant_status_topic": {
+          "type": "string",
+          "title": "Home Assistant status topic",
+          "description": "Home Assistant status topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant/status"]
+        },
+        "homeassistant_legacy_triggers": {
+          "type": "boolean",
+          "title": "Home Assistant legacy triggers",
+          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discovered",
+          "default": true
+        },
+        "soft_reset_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "requiresRestart": true,
+          "title": "Soft reset timeout (deprecated)",
+          "description": "Soft reset ZNP after timeout",
+          "readOnly": true
+        },
+        "report": {
+          "type": "boolean",
+          "title": "Reporting",
+          "requiresRestart": true,
+          "readOnly": true,
+          "description": "Enables report feature (deprecated)"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate (deprecated)",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS (deprecated)",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url (deprecated)",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
         }
       }
+    },
+    "experimental": {
+      "type": "object",
+      "title": "Experimental (deprecated)",
+      "properties": {
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        }
+      }
+    },
+    "whitelist": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Whitelist (deprecated, use passlist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "ban": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Ban (deprecated, use blocklist)",
+      "items": {
+        "type": "string"
+      }
     }
   },
   "required": ["mqtt"],

lib/util/settings.ts

@@ -5,33 +5,46 @@ import path from 'path';
 import yaml from './yaml';
 import Ajv from 'ajv';
 import schemaJson from './settings.schema.json';
-export const schema = schemaJson;
+export let schema = schemaJson;
+// @ts-ignore
+schema = {};
+objectAssignDeep(schema, schemaJson);
+
+// Remove legacy settings from schema
+{
+    delete schema.properties.advanced.properties.homeassistant_discovery_topic;
+    delete schema.properties.advanced.properties.homeassistant_legacy_entity_attributes;
+    delete schema.properties.advanced.properties.homeassistant_legacy_triggers;
+    delete schema.properties.advanced.properties.homeassistant_status_topic;
+    delete schema.properties.advanced.properties.soft_reset_timeout;
+    delete schema.properties.advanced.properties.report;
+    delete schema.properties.advanced.properties.baudrate;
+    delete schema.properties.advanced.properties.rtscts;
+    delete schema.properties.advanced.properties.ikea_ota_use_test_url;
+    delete schema.properties.experimental;
+    delete schemaJson.properties.whitelist;
+    delete schemaJson.properties.ban;
+}
 
 // DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
 const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
-const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schemaJson);
 const ajvRestartRequired = new Ajv({allErrors: true})
-    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+    .addKeyword({keyword: 'requiresRestart', validate: (s: unknown) => !s}).compile(schemaJson);
 
 const defaults: RecursivePartial<Settings> = {
-    passlist: [],
-    blocklist: [],
-    // Deprecated: use block/passlist
-    whitelist: [],
-    ban: [],
     permit_join: false,
+    external_converters: [],
     mqtt: {
+        base_topic: 'zigbee2mqtt',
         include_device_information: false,
-        /**
-         * Configurable force disable retain flag on mqtt publish.
-         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
-         */
         force_disable_retain: false,
     },
     serial: {
         disable_led: false,
     },
-    device_options: {},
+    passlist: [],
+    blocklist: [],
     map_options: {
         graphviz: {
             colors: {
@@ -52,12 +65,14 @@ const defaults: RecursivePartial<Settings> = {
             },
         },
     },
-    experimental: {
-        // json or attribute or attribute_and_json
-        output: 'json',
+    ota: {
+        update_check_interval: 24 * 60,
+        disable_automatic_update_check: false,
     },
+    device_options: {},
     advanced: {
         legacy_api: true,
+        legacy_availability_payload: true,
         log_rotation: true,
         log_symlink_current: false,
         log_output: ['console', 'file'],
@@ -65,106 +80,113 @@ const defaults: RecursivePartial<Settings> = {
         log_file: 'log.txt',
         log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
         log_syslog: {},
-        soft_reset_timeout: 0,
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         adapter_concurrent: null,
         adapter_delay: null,
-
-        // Availability timeout in seconds, disabled by default.
-        availability_blocklist: [],
-        availability_passlist: [],
-        // Deprecated, use block/passlist
-        availability_blacklist: [],
-        availability_whitelist: [],
-
-        /**
-         * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
-         * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
-         *
-         * Therefore Zigbee2MQTT BY DEFAULT caches all values and resend it with every message.
-         * advanced.cache_state in configuration.yaml allows to configure this.
-         * https://www.zigbee2mqtt.io/guide/configuration/
-         */
         cache_state: true,
         cache_state_persistent: true,
         cache_state_send_on_startup: true,
-
-        /**
-         * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
-         * "ISO_8601": ISO 8601 format
-         * "ISO_8601_local": Local ISO 8601 format (instead of UTC-based)
-         * "epoch": milliseconds elapsed since the UNIX epoch
-         * "disable": no last_seen attribute (default)
-         */
         last_seen: 'disable',
-
-        // Optional: Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg
         elapsed: false,
-
-        /**
-         * https://github.com/Koenkk/zigbee2mqtt/issues/685#issuecomment-449112250
-         *
-         * Network key will serve as the encryption key of your network.
-         * Changing this will require you to repair your devices.
-         */
         network_key: [1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13],
-
-        /**
-         * Enables reporting feature
-         */
-        report: false,
-
-        /**
-         * Home Assistant discovery topic
-         */
-        homeassistant_discovery_topic: 'homeassistant',
-
-        /**
-         * Home Assistant status topic
-         */
-        homeassistant_status_topic: 'hass/status',
-
-        /**
-         * Home Assistant legacy entity attributes, when enabled:
-         * Zigbee2MQTT will send additional states as attributes with each entity.
-         * For example, A temperature & humidity sensor will have 2 entities for
-         * the temperature and humidity, with this setting enabled both entities
-         * will also have an temperature and humidity attribute.
-         */
-        homeassistant_legacy_entity_attributes: true,
-
-        /**
-         * Home Assistant legacy triggers, when enabled:
-         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
-         * - A 'sensor_action' and 'sensor_click' will be discoverd
-         */
-        homeassistant_legacy_triggers: true,
-
-        /**
-         * Configurable timestampFormat
-         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
-         */
         timestamp_format: 'YYYY-MM-DD HH:mm:ss',
+        output: 'json',
+        // Everything below is deprecated
+        availability_blocklist: [],
+        availability_passlist: [],
+        availability_blacklist: [],
+        availability_whitelist: [],
+        soft_reset_timeout: 0,
+        report: false,
     },
-    ota: {
-        /**
-         * Minimal time delta in minutes between polling third party server for potential firmware updates
-         */
-        update_check_interval: 24 * 60,
-        /**
-         * Completely disallow Zigbee devices to initiate a search for a potential firmware update.
-         * If set to true, only a user-initiated update search will be possible.
-         */
-        disable_automatic_update_check: false,
-    },
-    external_converters: [],
 };
 
 let _settings: Partial<Settings>;
 let _settingsWithDefaults: Settings;
 
+function loadSettingsWithDefaults(): void {
+    _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
+
+    if (!_settingsWithDefaults.devices) {
+        _settingsWithDefaults.devices = {};
+    }
+
+    if (!_settingsWithDefaults.groups) {
+        _settingsWithDefaults.groups = {};
+    }
+
+    if (_settingsWithDefaults.homeassistant) {
+        const defaults = {discovery_topic: 'homeassistant', status_topic: 'hass/status',
+            legacy_entity_attributes: true, legacy_triggers: true};
+        const sLegacy = {};
+        if (_settingsWithDefaults.advanced) {
+            for (const key of ['homeassistant_legacy_triggers', 'homeassistant_discovery_topic',
+                'homeassistant_legacy_entity_attributes', 'homeassistant_status_topic']) {
+                // @ts-ignore
+                if (_settingsWithDefaults.advanced[key] !== undefined) {
+                    // @ts-ignore
+                    sLegacy[key.replace('homeassistant_', '')] = _settingsWithDefaults.advanced[key];
+                }
+            }
+        }
+
+        const s = typeof _settingsWithDefaults.homeassistant === 'object' ? _settingsWithDefaults.homeassistant : {};
+        // @ts-ignore
+        _settingsWithDefaults.homeassistant = {};
+        objectAssignDeep(_settingsWithDefaults.homeassistant, defaults, sLegacy, s);
+    }
+
+    if (_settingsWithDefaults.availability || _settingsWithDefaults.advanced?.availability_timeout) {
+        const defaults = {};
+        const s = typeof _settingsWithDefaults.availability === 'object' ? _settingsWithDefaults.availability : {};
+        // @ts-ignore
+        _settingsWithDefaults.availability = {};
+        objectAssignDeep(_settingsWithDefaults.availability, defaults, s);
+    }
+
+    if (_settingsWithDefaults.frontend) {
+        const defaults = {port: 8080, auth_token: false, host: '0.0.0.0'};
+        const s = typeof _settingsWithDefaults.frontend === 'object' ? _settingsWithDefaults.frontend : {};
+        // @ts-ignore
+        _settingsWithDefaults.frontend = {};
+        objectAssignDeep(_settingsWithDefaults.frontend, defaults, s);
+    }
+
+    if (_settings.advanced?.hasOwnProperty('baudrate') && _settings.serial?.baudrate == null) {
+        // @ts-ignore
+        _settingsWithDefaults.serial.baudrate = _settings.advanced.baudrate;
+    }
+
+    if (_settings.advanced?.hasOwnProperty('rtscts') && _settings.serial?.rtscts == null) {
+        // @ts-ignore
+        _settingsWithDefaults.serial.rtscts = _settings.advanced.rtscts;
+    }
+
+    if (_settings.advanced?.hasOwnProperty('ikea_ota_use_test_url') && _settings.ota?.ikea_ota_use_test_url == null) {
+        // @ts-ignore
+        _settingsWithDefaults.ota.ikea_ota_use_test_url = _settings.advanced.ikea_ota_use_test_url;
+    }
+
+    // @ts-ignore
+    if (_settings.experimental?.hasOwnProperty('transmit_power') && _settings.advanced?.transmit_power == null) {
+        // @ts-ignore
+        _settingsWithDefaults.advanced.transmit_power = _settings.experimental.transmit_power;
+    }
+
+    // @ts-ignore
+    if (_settings.experimental?.hasOwnProperty('output') && _settings.advanced?.output == null) {
+        // @ts-ignore
+        _settingsWithDefaults.advanced.output = _settings.experimental.output;
+    }
+
+    // @ts-ignore
+    _settingsWithDefaults.ban && _settingsWithDefaults.blocklist.push(..._settingsWithDefaults.ban);
+    // @ts-ignore
+    _settingsWithDefaults.whitelist && _settingsWithDefaults.passlist.push(..._settingsWithDefaults.whitelist);
+}
+
 function write(): void {
     const settings = getInternalSettings();
     const toWrite: KeyValue = objectAssignDeep({}, settings);
@@ -213,7 +235,7 @@ function write(): void {
     yaml.writeIfChanged(file, toWrite);
 
     _settings = read();
-    _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
+    loadSettingsWithDefaults();
 }
 
 export function validate(): string[] {
@@ -247,7 +269,7 @@ export function validate(): string[] {
 
     // Verify that all friendly names are unique
     const names: string[] = [];
-    const check = (e: DeviceSettings | GroupSettings): void => {
+    const check = (e: DeviceOptions | GroupOptions): void => {
         if (names.includes(e.friendly_name)) errors.push(`Duplicate friendly_name '${e.friendly_name}' found`);
         errors.push(...utils.validateFriendlyName(e.friendly_name));
         names.push(e.friendly_name);
@@ -376,7 +398,7 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
             }
         });
     };
-    iterate(schema.properties, []);
+    iterate(schemaJson.properties, []);
 }
 
 function getInternalSettings(): Partial<Settings> {
@@ -390,15 +412,7 @@ function getInternalSettings(): Partial<Settings> {
 
 export function get(): Settings {
     if (!_settingsWithDefaults) {
-        _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
-    }
-
-    if (!_settingsWithDefaults.devices) {
-        _settingsWithDefaults.devices = {};
-    }
-
-    if (!_settingsWithDefaults.groups) {
-        _settingsWithDefaults.groups = {};
+        loadSettingsWithDefaults();
     }
 
     return _settingsWithDefaults;
@@ -443,7 +457,7 @@ export function apply(newSettings: Record<string, unknown>): boolean {
     return restartRequired;
 }
 
-export function getGroup(IDorName: string | number): GroupSettings {
+export function getGroup(IDorName: string | number): GroupOptions {
     const settings = get();
     const byID = settings.groups[IDorName];
     if (byID) {
@@ -459,14 +473,14 @@ export function getGroup(IDorName: string | number): GroupSettings {
     return null;
 }
 
-export function getGroups(): GroupSettings[] {
+export function getGroups(): GroupOptions[] {
     const settings = get();
     return Object.entries(settings.groups).map(([ID, group]) => {
         return {devices: [], ...group, ID: Number(ID)};
     });
 }
 
-function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
+function getGroupThrowIfNotExists(IDorName: string): GroupOptions {
     const group = getGroup(IDorName);
     if (!group) {
         throw new Error(`Group '${IDorName}' does not exist`);
@@ -475,7 +489,7 @@ function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
     return group;
 }
 
-export function getDevice(IDorName: string): DeviceSettings {
+export function getDevice(IDorName: string): DeviceOptions {
     const settings = get();
     const byID = settings.devices[IDorName];
     if (byID) {
@@ -491,7 +505,7 @@ export function getDevice(IDorName: string): DeviceSettings {
     return null;
 }
 
-function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
+function getDeviceThrowIfNotExists(IDorName: string): DeviceOptions {
     const device = getDevice(IDorName);
     if (!device) {
         throw new Error(`Device '${IDorName}' does not exist`);
@@ -500,7 +514,7 @@ function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
     return device;
 }
 
-export function addDevice(ID: string): DeviceSettings {
+export function addDevice(ID: string): DeviceOptions {
     if (getDevice(ID)) {
         throw new Error(`Device '${ID}' already exists`);
     }
@@ -516,17 +530,17 @@ export function addDevice(ID: string): DeviceSettings {
     return getDevice(ID);
 }
 
-export function whitelistDevice(ID: string): void {
+export function addDeviceToPasslist(ID: string): void {
     const settings = getInternalSettings();
-    if (!settings.whitelist) {
-        settings.whitelist = [];
+    if (!settings.passlist) {
+        settings.passlist = [];
     }
 
-    if (settings.whitelist.includes(ID)) {
-        throw new Error(`Device '${ID}' already whitelisted`);
+    if (settings.passlist.includes(ID)) {
+        throw new Error(`Device '${ID}' already in passlist`);
     }
 
-    settings.whitelist.push(ID);
+    settings.passlist.push(ID);
     write();
 }
 
@@ -540,16 +554,6 @@ export function blockDevice(ID: string): void {
     write();
 }
 
-export function banDevice(ID: string): void {
-    const settings = getInternalSettings();
-    if (!settings.ban) {
-        settings.ban = [];
-    }
-
-    settings.ban.push(ID);
-    write();
-}
-
 export function removeDevice(IDorName: string): void {
     const device = getDeviceThrowIfNotExists(IDorName);
     const settings = getInternalSettings();
@@ -567,7 +571,7 @@ export function removeDevice(IDorName: string): void {
     write();
 }
 
-export function addGroup(name: string, ID?: string): GroupSettings {
+export function addGroup(name: string, ID?: string): GroupOptions {
     utils.validateFriendlyName(name, true);
     if (getGroup(name) || getDevice(name)) {
         throw new Error(`friendly_name '${name}' is already in use`);

lib/util/utils.ts

@@ -38,7 +38,7 @@ const endpointNames = [
     'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
     'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
     'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
-    'heat', 'cool', 'water', 'meter', 'wifi',
+    'heat', 'cool', 'water', 'meter', 'wifi', 'no_occupancy_since',
 ];
 
 function capitalize(s: string): string {
@@ -245,9 +245,13 @@ function sanitizeImageParameter(parameter: string): string {
     return sanitized;
 }
 
-function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boolean {
-    if (device.settings.hasOwnProperty('availability')) {
-        return !!device.settings.availability;
+function isAvailabilityEnabledForEntity(entity: Device | Group, settings: Settings): boolean {
+    if (entity.isGroup()) {
+        return !entity.membersDevices().map((d) => isAvailabilityEnabledForEntity(d, settings)).includes(false);
+    }
+
+    if (entity.options.hasOwnProperty('availability')) {
+        return !!entity.options.availability;
     }
 
     // availability_timeout = deprecated
@@ -256,11 +260,11 @@ function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boo
 
     const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
     if (passlist.length > 0) {
-        return passlist.includes(device.name) || passlist.includes(device.ieeeAddr);
+        return passlist.includes(entity.name) || passlist.includes(entity.ieeeAddr);
     }
 
     const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
-    return !blocklist.includes(device.name) && !blocklist.includes(device.ieeeAddr);
+    return !blocklist.includes(entity.name) && !blocklist.includes(entity.ieeeAddr);
 }
 
 const entityIDRegex = new RegExp(`^(.+?)(?:/(${endpointNames.join('|')}|\\d+))?$`);
@@ -277,6 +281,10 @@ function isZHGroup(obj: unknown): obj is zh.Group {
     return obj.constructor.name.toLowerCase() === 'group';
 }
 
+function availabilityPayload(state: 'online' | 'offline', settings: Settings): string {
+    return settings.advanced.legacy_availability_payload ? state : JSON.stringify({state});
+}
+
 const hours = (hours: number): number => 1000 * 60 * 60 * hours;
 const minutes = (minutes: number): number => 1000 * 60 * minutes;
 const seconds = (seconds: number): number => 1000 * seconds;
@@ -302,5 +310,5 @@ export default {
     equalsPartial, getObjectProperty, getResponse, parseJSON, loadModuleFromText, loadModuleFromFile,
     getExternalConvertersDefinitions, removeNullPropertiesFromObject, toNetworkAddressHex, toSnakeCase,
     parseEntityID, isEndpoint, isZHGroup, hours, minutes, seconds, validateFriendlyName, sleep,
-    sanitizeImageParameter, isAvailabilityEnabledForDevice, publishLastSeen,
+    sanitizeImageParameter, isAvailabilityEnabledForEntity, publishLastSeen, availabilityPayload,
 };

lib/zigbee.ts

@@ -36,8 +36,8 @@ export default class Zigbee {
             databaseBackupPath: data.joinPath('database.db.backup'),
             backupPath: data.joinPath('coordinator_backup.json'),
             serialPort: {
-                baudRate: settings.get().advanced.baudrate,
-                rtscts: settings.get().advanced.rtscts,
+                baudRate: settings.get().serial.baudrate,
+                rtscts: settings.get().serial.rtscts,
                 path: settings.get().serial.port,
                 adapter: settings.get().serial.adapter,
             },
@@ -112,8 +112,8 @@ export default class Zigbee {
 
         for (const device of this.devices(false)) {
             // If a passlist is used, all other device will be removed from the network.
-            const passlist = settings.get().passlist.concat(settings.get().whitelist);
-            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            const passlist = settings.get().passlist;
+            const blocklist = settings.get().blocklist;
             const remove = async (device: Device): Promise<void> => {
                 try {
                     await device.zh.removeFromNetwork();
@@ -133,8 +133,8 @@ export default class Zigbee {
         }
 
         // Check if we have to set a transmit power
-        if (settings.get().experimental.hasOwnProperty('transmit_power')) {
-            const transmitPower = settings.get().experimental.transmit_power;
+        if (settings.get().advanced.hasOwnProperty('transmit_power')) {
+            const transmitPower = settings.get().advanced.transmit_power;
             await this.herdsman.setTransmitPower(transmitPower);
             logger.info(`Set transmit power to '${transmitPower}'`);
         }
@@ -219,7 +219,7 @@ export default class Zigbee {
         }
     }
 
-    private resolveDevice(ieeeAddr: string): Device {
+    @bind private resolveDevice(ieeeAddr: string): Device {
         if (!this.deviceLookup[ieeeAddr]) {
             const device = this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
             device && (this.deviceLookup[ieeeAddr] = new Device(device));
@@ -235,7 +235,7 @@ export default class Zigbee {
     private resolveGroup(groupID: number): Group {
         const group = this.herdsman.getGroupByID(Number(groupID));
         if (group && !this.groupLookup[groupID]) {
-            this.groupLookup[groupID] = new Group(group);
+            this.groupLookup[groupID] = new Group(group, this.resolveDevice);
         }
 
         return this.groupLookup[groupID];
@@ -275,8 +275,8 @@ export default class Zigbee {
 
     @bind private async acceptJoiningDeviceHandler(ieeeAddr: string): Promise<boolean> {
         // If passlist is set, all devices not on passlist will be rejected to join the network
-        const passlist = settings.get().passlist.concat(settings.get().whitelist);
-        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        const passlist = settings.get().passlist;
+        const blocklist = settings.get().blocklist;
         if (passlist.length > 0) {
             if (passlist.includes(ieeeAddr)) {
                 logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.22.2",
+  "version": "1.25.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -35,54 +35,56 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "^8.8.2",
+    "ajv": "^8.11.0",
     "bind-decorator": "^1.0.11",
     "connect-gzip-static": "^2.1.1",
-    "core-js": "^3.20.1",
+    "core-js": "^3.21.1",
     "debounce": "^1.2.1",
-    "deep-object-diff": "^1.1.0",
+    "deep-object-diff": "^1.1.7",
     "fast-deep-equal": "^3.1.3",
-    "finalhandler": "^1.1.2",
+    "finalhandler": "^1.2.0",
     "git-last-commit": "^1.0.1",
     "humanize-duration": "^3.27.1",
     "js-yaml": "^4.1.0",
     "json-stable-stringify-without-jsonify": "^1.0.1",
     "mkdir-recursive": "^0.4.0",
     "moment": "^2.29.1",
-    "mqtt": "4.3.1",
+    "mqtt": "4.3.7",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
     "semver": "^7.3.5",
     "source-map-support": "^0.5.21",
-    "winston": "^3.3.3",
-    "winston-syslog": "^2.4.4",
-    "ws": "^8.4.0",
-    "zigbee-herdsman": "0.13.188",
-    "zigbee-herdsman-converters": "14.0.366",
-    "zigbee2mqtt-frontend": "0.6.67"
+    "uri-js": "^4.4.1",
+    "winston": "^3.6.0",
+    "winston-syslog": "^2.5.0",
+    "winston-transport": "^4.5.0",
+    "ws": "^8.5.0",
+    "zigbee-herdsman": "0.14.20",
+    "zigbee-herdsman-converters": "14.0.467",
+    "zigbee2mqtt-frontend": "0.6.80"
   },
   "devDependencies": {
-    "@babel/core": "^7.16.5",
-    "@babel/plugin-proposal-decorators": "^7.16.5",
-    "@babel/preset-env": "^7.16.5",
-    "@babel/preset-typescript": "^7.16.5",
+    "@babel/core": "^7.17.8",
+    "@babel/plugin-proposal-decorators": "^7.17.8",
+    "@babel/preset-env": "^7.16.11",
+    "@babel/preset-typescript": "^7.16.7",
     "@types/debounce": "^1.2.1",
     "@types/finalhandler": "^1.1.1",
     "@types/humanize-duration": "^3.27.1",
-    "@types/jest": "^27.0.3",
+    "@types/jest": "^27.4.1",
     "@types/js-yaml": "^4.0.5",
     "@types/object-assign-deep": "^0.4.0",
     "@types/rimraf": "^3.0.2",
-    "@types/ws": "^8.2.2",
-    "@typescript-eslint/eslint-plugin": "^5.8.0",
-    "@typescript-eslint/parser": "^5.8.0",
-    "babel-jest": "^27.4.5",
-    "eslint": "^8.5.0",
+    "@types/ws": "^8.5.3",
+    "@typescript-eslint/eslint-plugin": "^5.16.0",
+    "@typescript-eslint/parser": "^5.16.0",
+    "babel-jest": "^27.5.1",
+    "eslint": "^8.12.0",
     "eslint-config-google": "^0.14.0",
-    "eslint-plugin-jest": "^25.3.0",
-    "jest": "^27.4.5",
+    "eslint-plugin-jest": "^26.1.3",
+    "jest": "^27.5.1",
     "tmp": "^0.2.1",
-    "typescript": "^4.5.4"
+    "typescript": "^4.6.3"
   },
   "jest": {
     "coverageThreshold": {

scripts/generateNewSupportedDevices.js

@@ -6,5 +6,6 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
 for (const line of text.split('\n')) {
     const model = zigbeeHerdsmanConverters.devices.find((d) => d.model === line);
+    if (!model) throw new Error(`${line} does not exist`);
     console.log(`- \`${line}\` ${model.vendor} ${model.description}`);
 }

test/availability.test.js

@@ -227,9 +227,23 @@ describe('Availability', () => {
         expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
     });
 
+    it('Should to enable availabilty for just one device', async () => {
+        settings.set(['availability'], false);
+        settings.set(['devices', devices.bulb_color.ieeeAddr, 'availability'], true);
+
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(11));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
     it('Should retrieve device state when it reconnects', async () => {
         MQTT.publish.mockClear();
 
+        const device = controller.zigbee.resolveEntity(devices.bulb_color.ieeeAddr);
+        controller.state.set(device, {state: 'OFF'})
+
         const endpoint = devices.bulb_color.getEndpoint(1);
         endpoint.read.mockClear();
 
@@ -242,18 +256,15 @@ describe('Availability', () => {
         expect(endpoint.read).toHaveBeenCalledTimes(0);
         await advancedTime(utils.seconds(2));
 
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl',
-            ['colorMode', 'currentX', 'currentY', 'enhancedCurrentHue', 'currentSaturation', 'colorTemperature']);
 
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
         await flushPromises();
         endpoint.read.mockImplementationOnce(() => {throw new Error('')});
         await advancedTime(utils.seconds(3));
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
     });
 
     it('Should republish availability when device is renamed', async () => {
@@ -262,6 +273,8 @@ describe('Availability', () => {
         MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({from: 'bulb_color', to: 'bulb_new_name'}));
         await flushPromises();
 
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            null, {retain: true, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_new_name/availability',
             'online', {retain: true, qos: 0}, expect.any(Function));
         await advancedTime(utils.hours(12));
@@ -269,6 +282,16 @@ describe('Availability', () => {
             'offline', {retain: true, qos: 0}, expect.any(Function));
     });
 
+    it('Should publish availabiltiy payload in JSON format', async () => {
+        settings.set(['advanced', 'legacy_availability_payload'], false);
+        await resetExtension();
+        MQTT.publish.mockClear();
+        await advancedTime(utils.hours(26));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            stringify({state: 'offline'}), {retain: true, qos: 0}, expect.any(Function));
+    });
+
     it('Deprecated - should allow to block via advanced.availability_blocklist', async () => {
         settings.set(['advanced', 'availability_blocklist'], [devices.bulb_color.ieeeAddr]);
         await resetExtension();
@@ -295,4 +318,21 @@ describe('Availability', () => {
         await advancedTime(utils.minutes(12));
         expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
     });
+
+    it('Should publish availabilty for groups', async () => {
+        settings.set(['devices', devices.bulb_color_2.ieeeAddr, 'availability'], true);
+        await resetExtension();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        MQTT.publish.mockClear();
+        await advancedTime(utils.minutes(12));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+        MQTT.publish.mockClear();
+        devices.bulb_color_2.lastSeen = Date.now();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color_2});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
 });

test/configure.test.js

@@ -192,19 +192,7 @@ describe('Configure', () => {
         expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
     });
 
-    it('Should not configure when interviewing', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        delete device.meta.configured;
-        device.interviewing = true;
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        await zigbeeHerdsman.events.lastSeenChanged({device});
-        await flushPromises();
-        expectRemoteNotConfigured();
-        device.interviewing = false;
-    });
-
-    it('Should configure when not interviewCompleted', async () => {
+    it('Should not configure when interview not completed', async () => {
         const device = zigbeeHerdsman.devices.remote;
         delete device.meta.configured;
         device.interviewCompleted = false;
@@ -212,7 +200,7 @@ describe('Configure', () => {
         mockClear(device);
         await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
-        expectRemoteConfigured();
+        expectRemoteNotConfigured();
         device.interviewCompleted = true;
     });
 

test/controller.test.js

@@ -28,9 +28,10 @@ describe('Controller', () => {
         MQTT.restoreOnMock();
         zigbeeHerdsman.returnDevices.splice(0);
         mockExit = jest.fn();
+        data.writeDefaultConfiguration();
+        settings.reRead();
         controller = new Controller(jest.fn(), mockExit);
         mocksClear.forEach((m) => m.mockClear());
-        data.writeDefaultConfiguration();
         settings.reRead();
         data.writeDefaultState();
     });
@@ -180,14 +181,6 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
     });
 
-    it('Should remove non whitelisted devices on startup', async () => {
-        settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
-        await controller.start();
-        await flushPromises();
-        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
-        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
-    });
-
     it('Should remove device on blocklist on startup', async () => {
         settings.set(['blocklist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -196,14 +189,6 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
     });
 
-    it('Should remove banned devices on startup', async () => {
-        settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
-        await controller.start();
-        await flushPromises();
-        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
-    });
-
     it('Start controller fails', async () => {
         zigbeeHerdsman.start.mockImplementationOnce(() => {throw new Error('failed')});
         await controller.start();
@@ -630,11 +615,13 @@ describe('Controller', () => {
     });
 
     it('Should disable legacy options on new network start', async () => {
-        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        settings.set(['homeassistant'], true);
+        settings.reRead();
+        expect(settings.get().homeassistant.legacy_entity_attributes).toBeTruthy();
         expect(settings.get().advanced.legacy_api).toBeTruthy();
         zigbeeHerdsman.start.mockReturnValueOnce('reset');
         await controller.start();
-        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().homeassistant.legacy_entity_attributes).toBeFalsy();
         expect(settings.get().advanced.legacy_api).toBeFalsy();
     });
 

test/homeassistant.test.js

@@ -24,6 +24,7 @@ describe('HomeAssistant extension', () => {
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings.reRead();
+        settings.set(['homeassistant'], true);
         data.writeEmptyState();
         controller.state.load();
         await resetExtension();
@@ -50,14 +51,19 @@ describe('HomeAssistant extension', () => {
         const duplicated = [];
         require('zigbee-herdsman-converters').devices.forEach((d) => {
             const exposes = typeof d.exposes == 'function' ? d.exposes() : d.exposes;
-            const device = {definition: d, isDevice: () => true, settings: {}, exposes: () => exposes};
+            const device = {definition: d, isDevice: () => true, options: {}, exposes: () => exposes};
             const configs = extension.getConfigs(device);
             const cfg_type_object_ids = [];
 
             configs.forEach((c) => {
                 const id = c['type'] + '/' + c['object_id'];
                 if (cfg_type_object_ids.includes(id)) {
-                    duplicated.push(d.model);
+                    if (typeof d.exposes == 'function') {
+                        // A dynamic function must exposes all possible attributes for the docs
+                        console.warn(`${d.model} dynamic exposes contains duplicated ${id}`)
+                    } else {
+                        duplicated.push(d.model);
+                    }
                 } else {
                     cfg_type_object_ids.push(id);
                 }
@@ -475,7 +481,7 @@ describe('HomeAssistant extension', () => {
                 device: {
                     manufacturer: 'Not from Xiaomi',
                     model: 'custom model',
-                }
+                },
             },
             friendly_name: 'weather_sensor',
             retain: false,
@@ -544,6 +550,75 @@ describe('HomeAssistant extension', () => {
         );
     });
 
+    it('Should discover devices with overriden name', async () => {
+        settings.set(['devices', '0x0017880104e45522'], {
+            homeassistant: {
+                name: "Weather Sensor",
+            },
+            friendly_name: 'weather_sensor',
+            retain: false,
+        })
+
+        await resetExtension();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'state_class': 'measurement',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor',
+            'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
+            'name': 'Weather Sensor temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'Weather Sensor',
+                'sw_version': null,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            'enabled_by_default': true,
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            'unit_of_measurement': '%',
+            'device_class': 'humidity',
+            'state_class': 'measurement',
+            'value_template': '{{ value_json.humidity }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor',
+            'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
+            'name': 'Weather Sensor humidity',
+            'unique_id': '0x0017880104e45522_humidity_zigbee2mqtt',
+            'enabled_by_default': true,
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'Weather Sensor',
+                'sw_version': null,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/humidity/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
     it('Should discover devices with overriden user configuration affecting type and object_id', async () => {
         settings.set(['devices', '0x0017880104e45541'], {
             friendly_name: 'my_switch',
@@ -555,8 +630,7 @@ describe('HomeAssistant extension', () => {
                 light: {
                     type: 'this should be ignored',
                     name: 'my_light_name_override'
-                }
-
+                },
             },
         })
 
@@ -671,29 +745,35 @@ describe('HomeAssistant extension', () => {
         let payload;
 
         payload = {
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            "action_template":"{% set values = {None:None,'idle':'off','heat':'heating','cool':'cooling','fan_only':'fan'} %}{{ values[value_json.running_state] }}",
+            "action_topic":"zigbee2mqtt/TS0601_thermostat",
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
             "away_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/away_mode",
             "away_mode_state_template":"{{ value_json.away_mode }}",
             "away_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "current_temperature_template":"{{ value_json.local_temperature }}",
             "current_temperature_topic":"zigbee2mqtt/TS0601_thermostat",
             "device":{
-               "identifiers":[
-                  "zigbee2mqtt_0x0017882104a44559"
-               ],
-               "manufacturer":"TuYa",
-               "model":"Radiator valve with thermostat (TS0601_thermostat)",
-               "name":"TS0601_thermostat",
-               "sw_version":  null,
+                "identifiers":[
+                    "zigbee2mqtt_0x0017882104a44559"
+                ],
+                "manufacturer":"TuYa",
+                "model":"Radiator valve with thermostat (TS0601_thermostat)",
+                "name":"TS0601_thermostat",
+                "sw_version":null
             },
             "hold_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
             "hold_modes":[
-               "schedule",
-               "manual",
-               "boost",
-               "complex",
-               "comfort",
-               "eco"
+                "schedule",
+                "manual",
+                "boost",
+                "complex",
+                "comfort",
+                "eco"
             ],
             "hold_state_template":"{{ value_json.preset }}",
             "hold_state_topic":"zigbee2mqtt/TS0601_thermostat",
@@ -704,7 +784,9 @@ describe('HomeAssistant extension', () => {
             "mode_state_template":"{{ value_json.system_mode }}",
             "mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "modes":[
-               "heat", "auto", "off"
+                "heat",
+                "auto",
+                "off"
             ],
             "name":"TS0601_thermostat",
             "temp_step":0.5,
@@ -924,6 +1006,19 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
     });
 
+    it('Should discover when options change', async () => {
+        const device = controller.zigbee.resolveEntity(zigbeeHerdsman.devices.bulb);
+        MQTT.publish.mockClear();
+        controller.eventBus.emitEntityOptionsChanged({entity: device, from: {}, to: {'test': 123}});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            `homeassistant/light/${device.ID}/light/config`,
+            expect.any(String),
+            expect.any(Object),
+            expect.any(Function),
+        );
+    });
+
     it('Should send all status when home assistant comes online (default topic)', async () => {
         data.writeDefaultState();
         extension.state.load();
@@ -1141,6 +1236,50 @@ describe('HomeAssistant extension', () => {
         );
     });
 
+    it('Should refresh discovery when group is renamed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/rename', stringify({"from": "ha_discovery_group", "to": "ha_discovery_group_new","homeassistant_rename":true}));
+        await flushPromises();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group_new/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group_new",
+               "sw_version": version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group_new",
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "name":"ha_discovery_group_new",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group_new",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
     it('Shouldnt refresh discovery when device is renamed and homeassistant_rename is false', async () => {
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":false}));
@@ -1650,6 +1789,42 @@ describe('HomeAssistant extension', () => {
         );
     });
 
+    it('Should discover with json availability payload value_template', async () => {
+        settings.set(['advanced', 'legacy_availability_payload'], false);
+        await resetExtension();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state","value_template":'{{ value_json.state }}'}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version": version,
+            },
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
     it('Should discover last_seen when enabled', async () => {
         settings.set(['advanced', 'last_seen'], 'ISO_8601');
         await resetExtension();

test/legacy/bridgeLegacy.test.js

@@ -56,7 +56,7 @@ describe('Bridge legacy', () => {
     it('Should allow whitelist', async () => {
         const bulb_color = zigbeeHerdsman.devices.bulb_color;
         const bulb = zigbeeHerdsman.devices.bulb;
-        expect(settings.get().whitelist).toStrictEqual([]);
+        expect(settings.get().passlist).toStrictEqual([]);
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb_color');
         await flushPromises();
@@ -68,7 +68,7 @@ describe('Bridge legacy', () => {
         );
 
         MQTT.publish.mockClear()
-        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr]);
+        expect(settings.get().passlist).toStrictEqual([bulb_color.ieeeAddr]);
         MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -79,10 +79,10 @@ describe('Bridge legacy', () => {
         );
 
         MQTT.publish.mockClear()
-        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        expect(settings.get().passlist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
         MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
         await flushPromises();
-        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        expect(settings.get().passlist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
@@ -366,7 +366,7 @@ describe('Bridge legacy', () => {
         controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
         const device = zigbeeHerdsman.devices.bulb_color;
         device.removeFromNetwork.mockClear();
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
         await flushPromises();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
@@ -381,14 +381,14 @@ describe('Bridge legacy', () => {
             expect.any(Function)
         );
         expect(controller.state.state).toStrictEqual({});
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
     });
 
     it('Should allow to force remove device', async () => {
         controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
         const device = zigbeeHerdsman.devices.bulb_color;
         device.removeFromDatabase.mockClear();
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
         await flushPromises();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/force_remove', 'bulb_color');
@@ -403,13 +403,13 @@ describe('Bridge legacy', () => {
             expect.any(Function)
         );
         expect(controller.state.state).toStrictEqual({});
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
     });
 
-    it('Should allow to ban device', async () => {
+    it('Should allow to block device', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         device.removeFromNetwork.mockClear();
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
         await flushPromises();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
@@ -423,7 +423,7 @@ describe('Bridge legacy', () => {
             {qos: 0, retain: false},
             expect.any(Function)
         );
-        expect(settings.get().ban).toStrictEqual(['0x000b57fffec6a5b3']);
+        expect(settings.get().blocklist).toStrictEqual(['0x000b57fffec6a5b3']);
     });
 
     it('Shouldnt crash when removing non-existing device', async () => {

test/otaUpdate.test.js

@@ -7,10 +7,18 @@ const Controller = require('../lib/controller');
 const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const stringify = require('json-stable-stringify-without-jsonify');
+const zigbeeOTA = require('zigbee-herdsman-converters/lib/ota/zigbeeOTA');
+
+const spyUseIndexOverride = jest.spyOn(zigbeeOTA, 'useIndexOverride');
 
 describe('OTA update', () => {
     let controller;
 
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'OTAUpdate');
+        await controller.enableDisableExtension(true, 'OTAUpdate');
+    }
+
     const mockClear = (mapped) => {
         mapped.ota.updateToLatest = jest.fn();
         mapped.ota.isUpdateAvailable = jest.fn();
@@ -20,7 +28,7 @@ describe('OTA update', () => {
         data.writeDefaultConfiguration();
         settings.reRead();
         data.writeDefaultConfiguration();
-        settings.set(['advanced', 'ikea_ota_use_test_url'], true);
+        settings.set(['ota', 'ikea_ota_use_test_url'], true);
         settings.reRead();
         jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
@@ -70,9 +78,11 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+        expect(logger.info).toHaveBeenCalledWith(`Device 'bulb' was updated from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190103","softwareBuildID":3}'`);
         expect(device.save).toHaveBeenCalledTimes(2);
-        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhenActive': false});
+        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhen': 'immediate'});
+        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhen': 'active'});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
             stringify({"update_available":false,"update":{"state":"updating","progress":0}}),
@@ -90,7 +100,7 @@ describe('OTA update', () => {
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/update',
-            stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
+            stringify({"data":{"from":{"date_code":"20190101","software_build_id":1},"id":"bulb","to":{"date_code":"20190103","software_build_id":3}},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -369,10 +379,11 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+        expect(logger.info).toHaveBeenCalledWith(`Device 'bulb' was updated from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190103","softwareBuildID":3}'`);        
         expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(2);
-        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhenActive': false});
+        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhen': 'immediate'});
     });
 
     it('Legacy api: Should handle when OTA update fails', async () => {
@@ -453,4 +464,16 @@ describe('OTA update', () => {
         await flushPromises();
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
     });
+
+    it('Set zigbee_ota_override_index_location', async () => {
+        settings.set(['ota', 'zigbee_ota_override_index_location'], 'local.index.json');
+        await resetExtension();
+        expect(spyUseIndexOverride).toHaveBeenCalledWith(data.mockDir + '/local.index.json');
+        spyUseIndexOverride.mockClear();
+        
+        settings.set(['ota', 'zigbee_ota_override_index_location'], 'http://my.site/index.json');
+        await resetExtension();
+        expect(spyUseIndexOverride).toHaveBeenCalledWith('http://my.site/index.json');
+        spyUseIndexOverride.mockClear();
+    });
 });

test/publish.test.js

@@ -367,7 +367,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/thermostat_group/set', stringify({child_lock: 'LOCK'}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
-        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "dataRequest", {data: [1], datatype: 1, dp: 7, length_hi: 0, length_lo: 1, seq: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "dataRequest", {dpValues: [{data: [1], datatype: 1, dp: 7}], seq: expect.any(Number)}, {disableDefaultResponse: true});
     });
 
     it('Should publish messages to groups with on and brightness', async () => {
@@ -907,33 +907,6 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColor", {"colorx": 45940, "colory": 19595, "transtime": 30}, {}]);
     });
 
-    it('Should correctly toggle when transition is used', async () => {
-        settings.set(['devices', zigbeeHerdsman.devices.bulb_color.ieeeAddr, 'transition'], 20);
-        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON', brightness: 163}));
-        await flushPromises();
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
-        await flushPromises();
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'TOGGLE',}));
-        await flushPromises();
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'TOGGLE',}));
-        await flushPromises();
-        expect(endpoint.command).toHaveBeenCalledTimes(4);
-        expect(endpoint.command).toHaveBeenNthCalledWith(1, "genLevelCtrl", "moveToLevelWithOnOff", {"level": 163, "transtime": 200}, {})
-        expect(endpoint.command).toHaveBeenNthCalledWith(2, "genLevelCtrl", "moveToLevelWithOnOff", {"level": 0, "transtime": 200}, {})
-        expect(endpoint.command).toHaveBeenNthCalledWith(3, "genLevelCtrl", "moveToLevelWithOnOff", {"level": 163, "transtime": 200}, {})
-        expect(endpoint.command).toHaveBeenNthCalledWith(4, "genLevelCtrl", "moveToLevelWithOnOff", {"level": 0, "transtime": 200}, {})
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
-        expect(MQTT.publish).toHaveBeenNthCalledWith(1, 'zigbee2mqtt/bulb_color', stringify({"brightness":163,"state":"ON"}),
-            { retain: false, qos: 0 }, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenNthCalledWith(2, 'zigbee2mqtt/bulb_color', stringify({"brightness":163,"state":"OFF"}),
-            { retain: false, qos: 0 }, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenNthCalledWith(3, 'zigbee2mqtt/bulb_color', stringify({"brightness":163,"state":"ON"}),
-            { retain: false, qos: 0 }, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenNthCalledWith(4, 'zigbee2mqtt/bulb_color', stringify({"brightness":163,"state":"OFF"}),
-            { retain: false, qos: 0 }, expect.any(Function));
-    });
-
     it('Should turn device off when brightness 0 is send', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -1321,7 +1294,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "dataRequest", {data: [1], datatype: 1, dp: 3, length_hi: 0, length_lo: 1, seq: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "dataRequest", {dpValues: [{data: [1], datatype: 1, dp: 3}], seq: expect.any(Number)}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});

test/receive.test.js

@@ -254,42 +254,6 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-    it('Should handle a zigbee message with voltage 2850', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2850}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2650', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2650}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2000', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2000}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
     it('Should publish 1 message when converted twice', async () => {
         const device = zigbeeHerdsman.devices.RTCGQ11LM;
         const data = {'65281': {'1': 3045, '3': 19, '4': 17320, '5': 35, '6': [0, 3], '10': 51107, '11': 381, '100': 0}}
@@ -350,20 +314,6 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-    it('Should handle forwarded Xiaomi messages', async () => {
-        const device = zigbeeHerdsman.devices.ZNCZ02LM;
-        const payload = {data: {measuredValue: -85}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 6539};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/weather_sensor',
-            stringify({temperature: -0.85}),
-            {"qos": 1, "retain": false},
-            expect.any(Function),
-        )
-    });
-
     it('Should handle messages from Xiaomi router devices', async () => {
         const device = zigbeeHerdsman.devices.ZNCZ02LM;
         const data = {onOff: 1};

test/settings.test.js

@@ -66,7 +66,7 @@ describe('Settings', () => {
     it('Should apply environment variables', () => {
         process.env['ZIGBEE2MQTT_CONFIG_SERIAL_DISABLE_LED'] = 'true';
         process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_SOFT_RESET_TIMEOUT'] = 1;
-        process.env['ZIGBEE2MQTT_CONFIG_EXPERIMENTAL_OUTPUT'] = 'csvtest';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_OUTPUT'] = 'csvtest';
         process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
         process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
 
@@ -77,7 +77,7 @@ describe('Settings', () => {
         expected.groups = {};
         expected.serial.disable_led = true;
         expected.advanced.soft_reset_timeout = 1;
-        expected.experimental.output = 'csvtest';
+        expected.advanced.output = 'csvtest';
         expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
         expected.mqtt.base_topic = 'testtopic';
 
@@ -150,6 +150,7 @@ describe('Settings', () => {
         write(configurationFile, contentConfiguration);
 
         const expected = {
+            base_topic: 'zigbee2mqtt',
             include_device_information: false,
             force_disable_retain: false,
             password: "mysecretpassword",
@@ -672,14 +673,6 @@ describe('Settings', () => {
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
-    it('Should ban devices', () => {
-        write(configurationFile, {});
-        settings.banDevice('0x123');
-        expect(settings.get().ban).toStrictEqual(['0x123']);
-        settings.banDevice('0x1234');
-        expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
-    });
-
     it('Should add devices to blocklist', () => {
         write(configurationFile, {});
         settings.blockDevice('0x123');
@@ -842,4 +835,78 @@ describe('Settings', () => {
         const after = fs.statSync(configurationFile).mtimeMs;
         expect(before).toBe(after);
     });
+
+    it('Frontend config', () => {
+        write(configurationFile, {...minimalConfig,
+            frontend: true, 
+        });
+
+        settings.reRead();
+        expect(settings.get().frontend).toStrictEqual({port: 8080, auth_token: false, host: '0.0.0.0'})
+    });
+
+    it('Baudrate config', () => {
+        write(configurationFile, {...minimalConfig,
+            advanced: {baudrate: 20}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().serial.baudrate).toStrictEqual(20)
+    });
+
+    it('ikea_ota_use_test_url config', () => {
+        write(configurationFile, {...minimalConfig,
+            advanced: {ikea_ota_use_test_url: true}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().ota.ikea_ota_use_test_url).toStrictEqual(true)
+    });
+
+    it('transmit_power config', () => {
+        write(configurationFile, {...minimalConfig,
+            experimental: {transmit_power: 1337}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().advanced.transmit_power).toStrictEqual(1337)
+    });
+
+    it('output config', () => {
+        write(configurationFile, {...minimalConfig,
+            experimental: {output: 'json'}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().advanced.output).toStrictEqual('json')
+    });
+
+    it('Baudrartsctste config', () => {
+        write(configurationFile, {...minimalConfig,
+            advanced: {rtscts: true}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().serial.rtscts).toStrictEqual(true)
+    });
+
+    it('Deprecated: Home Assistant config', () => {
+        write(configurationFile, {...minimalConfig,
+            homeassistant: {discovery_topic: 'new'}, 
+            advanced: {homeassistant_discovery_topic: 'old', homeassistant_status_topic: 'olds'},
+        });
+
+        settings.reRead();
+        expect(settings.get().homeassistant).toStrictEqual({discovery_topic: 'new', legacy_entity_attributes: true, legacy_triggers: true, status_topic: 'olds'})
+    });
+
+    it('Deprecated: ban/whitelist config', () => {
+        write(configurationFile, {...minimalConfig,
+            ban: ['ban'], whitelist: ['whitelist'], passlist: ['passlist'], blocklist: ['blocklist']
+        });
+
+        settings.reRead();
+        expect(settings.get().blocklist).toStrictEqual(['blocklist', 'ban'])
+        expect(settings.get().passlist).toStrictEqual(['passlist', 'whitelist'])
+    });
 });

test/stub/data.js

@@ -21,7 +21,8 @@ function writeDefaultConfiguration() {
         devices: {
             "0x000b57fffec6a5b2": {
                 retain: true,
-                friendly_name: "bulb"
+                friendly_name: "bulb",
+                description: "this is my bulb",
             },
             "0x0017880104e45517": {
                 retain: true,

test/stub/logger.js

@@ -23,6 +23,7 @@ const mock = {
     setLevel: (newLevel) => {level = newLevel},
     getLevel: () => level,
     setTransportsEnabled: (value) => {transportsEnabled = value},
+    end: jest.fn(),
 };
 
 jest.mock('../../lib/util/logger', () => (mock));