1.29.1

This reverts commit 56f7f3e1bd.

.eslintrc.js

@@ -15,6 +15,7 @@ module.exports = {
         'indent': ['error', 4],
         'max-len': ['error', {'code': 120}],
         'no-prototype-builtins': 'off',
+        'linebreak-style': ['error', (process.platform === 'win32' ? 'windows' : 'unix')], // https://stackoverflow.com/q/39114446/2771889
     },
     'plugins': [
         'jest',

.github/ISSUE_TEMPLATE/new_device_support.yaml

@@ -8,6 +8,7 @@ body:
       value: |
         **IMPORTANT:** Before submitting:
         - Make sure this device is not already supported in the [dev branch](https://www.zigbee2mqtt.io/advanced/more/switch-to-dev-branch.html)
+        - Make sure there is no existing issue or PR for this device already, search for your device here: https://github.com/Koenkk/zigbee2mqtt/issues
         - Follow this [guide](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html)
         - If you are using the Home Assistant addon and are still on 1.18.1, check the first point of [this](https://github.com/Koenkk/zigbee2mqtt/releases/tag/1.19.0)
   - type: input

.github/ISSUE_TEMPLATE/problem_report.yaml

@@ -37,7 +37,7 @@ body:
     id: z2m_version
     attributes:
       label: Zigbee2MQTT version
-      description: Can be found in the frontend -> settings -> about -> Zigbee2MQTT version
+      description: Can be found in the frontend -> settings -> about -> Zigbee2MQTT version. Are you running Zigbee2MQTT 1.18.1? Then read [this](https://github.com/Koenkk/zigbee2mqtt/releases/tag/1.19.0).
       placeholder: '1.22.1'
     validations:
       required: true

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    target-branch: dev
+
+  - package-ecosystem: "docker"
+    directory: "/docker"
+    schedule:
+      interval: "daily"
+    target-branch: dev

.github/workflows/ci.yml

@@ -2,27 +2,20 @@ name: CI
 
 on: [push, pull_request]
 
+permissions:
+  contents: read
+
 jobs:
   ci:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
       with:
-        node-version: 14
+        node-version: 18
         registry-url: https://registry.npmjs.org/
-    - name: Get NodeJS Version
-      id: nodeversion
-      run: |
-        echo "::set-output name=version::$(node --version)"
-    - name: Restore node_modules cache
-      id: cache-node-modules
-      uses: actions/cache@v2
-      with:
-        path: node_modules
-        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+        cache: 'npm'
     - name: Install dependencies
-      if: steps.cache-node-modules.outputs.cache-hit != 'true'
       run: npm ci
     - name: Build
       run: npm run build
@@ -38,13 +31,13 @@ jobs:
       run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u koenkk --password-stdin
     - name: Docker setup - QEMU
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
       with:
         platforms: all
     - name: Docker setup - Buildx
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
       id: buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
       with:
         version: latest
     - name: Docker build dev
@@ -52,7 +45,7 @@ jobs:
       run: |
         docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          --platform linux/arm64/v8,linux/386,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
           -t koenkk/zigbee2mqtt:latest-dev -t ghcr.io/koenkk/zigbee2mqtt:latest-dev \
@@ -63,7 +56,7 @@ jobs:
         TAG="$(git describe --tags)"
         docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          --platform linux/arm64/v8,linux/386,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
           -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" -t ghcr.io/koenkk/zigbee2mqtt:latest -t "ghcr.io/koenkk/zigbee2mqtt:$TAG" \
@@ -76,9 +69,33 @@ jobs:
     - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
       if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
       run: |
-        curl -s \
-          --request POST \
-          --header "Content-Type: application/json" \
-          --data '{"definition": {"id": 9}}' \
-          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
-          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"
+        curl \
+          -X POST \
+          -H "Authorization: token ${{ secrets.CR_PAT }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          https://api.github.com/repos/zigbee2mqtt/hassio-zigbee2mqtt/actions/workflows/ci.yml/dispatches \
+          -d '{"ref":"master","inputs":{}}'
+
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        node: [14, 16, 18, 19]
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: https://registry.npmjs.org/
+          cache: 'npm'
+      - name: Install dependencies
+        # --ignore-scripts prevents the serialport build which often fails on Windows
+        run: npm ci --ignore-scripts
+      - name: Lint
+        run: npm run eslint
+      - name: Build
+        run: npm run build
+      - name: Test
+        run: npm run test-with-coverage

.github/workflows/codeql.yaml

@@ -0,0 +1,34 @@
+name: "Code Scanning - Action"
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  CodeQL-Build:
+    # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
+    runs-on: ubuntu-latest
+
+    permissions:
+      # required for all workflows
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+
+      # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below).
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      #- run: |
+      #     make bootstrap
+      #     make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/depsreview.yaml

@@ -0,0 +1,14 @@
+name: 'Dependency review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency review'
+        uses: actions/dependency-review-action@v3

.github/workflows/ghcr_cleanup.yml

@@ -0,0 +1,34 @@
+on:
+  workflow_dispatch:
+
+name: GHCR cleanup
+
+permissions: {}
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Delete untagged images
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.CR_PAT }}
+          script: |
+            const response = await github.request("GET /${{ env.OWNER }}/packages/container/${{ env.PACKAGE_NAME }}/versions",
+              { per_page: ${{ env.PER_PAGE }}
+            });
+            for(version of response.data) {
+                if (version.metadata.container.tags.length == 0) {
+                    try {
+                        console.log("delete " + version.id)
+                        const deleteResponse = await github.request("DELETE /${{ env.OWNER }}/packages/container/${{ env.PACKAGE_NAME }}/versions/" + version.id, { });
+                        console.log("status " + deleteResponse.status)
+                    } catch (e) {
+                        console.log("failed")
+                    }
+                }
+            }
+        env:
+          OWNER: user
+          PACKAGE_NAME: zigbee2mqtt
+          PER_PAGE: 2000

.github/workflows/stale.yml

@@ -4,11 +4,17 @@ on:
   - cron: "0 0 * * *"
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   stale:
+    permissions:
+      issues: write  # for actions/stale to close stale issues
+      pull-requests: write  # for actions/stale to close stale PRs
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v7
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
@@ -16,4 +22,4 @@ jobs:
         days-before-stale: 30
         days-before-close: 7
         operations-per-run: 500
-        exempt-issue-labels: 'breaking change'
+        exempt-issue-labels: dont-stale

.github/workflows/update_deps.yml

@@ -4,21 +4,26 @@ on:
   schedule:
   - cron: "0 0 * * 0"
 
+permissions:
+  contents: read
+
 jobs:
   update_deps:
+    permissions:
+      contents: write  # for peter-evans/create-pull-request to create branch
+      pull-requests: write  # for peter-evans/create-pull-request to create a PR
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         ref: dev
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
       with:
-        node-version: 14
-    - run: rm -rf npm-shrinkwrap.json
-    - run: npx npm-check-updates -u
-    - run: npm install && npm shrinkwrap --dev
-    - uses: peter-evans/create-pull-request@v3
+        node-version: 18
+    - run: npx npm-check-updates -u -x connect-gzip-static # connect-gzip-static only supports node 16
+    - run: npm install
+    - uses: peter-evans/create-pull-request@v4
       with:
         commit-message: Update dependencies
         branch: "update_deps"

.github/workflows/update_frontend.yml

@@ -3,26 +3,31 @@ on:
   repository_dispatch:
     types: update_frontend
 
+permissions: {}
 jobs:
   update_frontend:
+    permissions:
+      contents: write  #  to create branch (peter-evans/create-pull-request)
+      pull-requests: write  #  to create a PR (peter-evans/create-pull-request)
+
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         ref: dev
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
       with:
-        node-version: 14
+        node-version: 18
     - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v3
+    - uses: peter-evans/create-pull-request@v4
       id: cpr
       with:
         commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
         branch: "update_frontend"
         title: Update zigbee-frontend to ${{ github.event.client_payload.version }}
     - run: sleep 5 # Otherwise pull request may not exist yet causing automerge to fail
-    - uses: peter-evans/enable-pull-request-automerge@v1
+    - uses: peter-evans/enable-pull-request-automerge@v2
       if: steps.cpr.outputs.pull-request-operation == 'created'
       with:
         token: ${{ secrets.CR_PAT }}

.github/workflows/update_zh.yml

@@ -3,26 +3,31 @@ on:
   repository_dispatch:
     types: update_zh
 
+permissions: {}
 jobs:
   update_zh:
+    permissions:
+      contents: write  #  to create branch (peter-evans/create-pull-request)
+      pull-requests: write  #  to create a PR (peter-evans/create-pull-request)
+
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         ref: dev
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
       with:
-        node-version: 14
+        node-version: 18
     - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v3
+    - uses: peter-evans/create-pull-request@v4
       id: cpr
       with:
         commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
         branch: "update_zh"
         title: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
     - run: sleep 5 # Otherwise pull request may not exist yet causing automerge to fail
-    - uses: peter-evans/enable-pull-request-automerge@v1
+    - uses: peter-evans/enable-pull-request-automerge@v2
       if: steps.cpr.outputs.pull-request-operation == 'created'
       with:
         token: ${{ secrets.CR_PAT }}

.github/workflows/update_zhc.yml

@@ -3,26 +3,31 @@ on:
   repository_dispatch:
     types: update_zhc
 
+permissions: {}
 jobs:
   update_zhc:
+    permissions:
+      contents: write  #  to create branch (peter-evans/create-pull-request)
+      pull-requests: write  #  to create a PR (peter-evans/create-pull-request)
+
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         ref: dev
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
       with:
-        node-version: 14
+        node-version: 18
     - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v3
+    - uses: peter-evans/create-pull-request@v4
       id: cpr
       with:
         commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
         branch: "update_zhc"
         title: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
     - run: sleep 5 # Otherwise pull request may not exist yet causing automerge to fail
-    - uses: peter-evans/enable-pull-request-automerge@v1
+    - uses: peter-evans/enable-pull-request-automerge@v2
       if: steps.cpr.outputs.pull-request-operation == 'created'
       with:
         token: ${{ secrets.CR_PAT }}

README.md

@@ -54,7 +54,7 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 
 ### [Home Assistant](https://www.home-assistant.io/)
 - [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/zigbee2mqtt/hassio-zigbee2mqtt)
-- Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/guide/usage/integrations/home_assistant.html)
+- Generic install: using instructions [here](https://www.zigbee2mqtt.io/guide/usage/integrations/home_assistant.html)
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">
 

docker/Dockerfile

@@ -1,29 +1,26 @@
-FROM node:14-alpine3.12 as base
+FROM alpine:3.17.0 as base
 
 WORKDIR /app
-RUN apk add --no-cache tzdata eudev tini
-
-COPY package.json ./
+RUN apk add --no-cache tzdata eudev tini nodejs
 
 # Dependencies and build
 FROM base as dependencies_and_build
 
-COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY package*.json tsconfig.json index.js ./
 COPY lib ./lib
 
-RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
-    npm ci --no-audit --no-optional --no-update-notifier && \
-    npm run build && \
-    rm -rf node_modules && \
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git npm && \
     npm ci --production --no-audit --no-optional --no-update-notifier && \
+    # Serialport needs to be rebuild for Alpine https://serialport.io/docs/9.x.x/guide-installation#alpine-linux
+    npm rebuild --build-from-source && \
     apk del .buildtools
 
 # Release
 FROM base as release
 
 COPY --from=dependencies_and_build /app/node_modules ./node_modules
-COPY --from=dependencies_and_build /app/dist ./dist
-COPY LICENSE index.js data/configuration.yaml ./
+COPY dist ./dist
+COPY package.json LICENSE index.js data/configuration.yaml ./
 
 COPY docker/docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
@@ -33,5 +30,7 @@ RUN mkdir /app/data
 ARG COMMIT
 RUN echo "$COMMIT" > dist/.hash
 
+ENV NODE_ENV production
+
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD [ "/sbin/tini", "--", "node", "index.js"]

index.js

@@ -1,9 +1,7 @@
-require('core-js/features/object/from-entries');
-require('core-js/features/array/flat');
 const semver = require('semver');
 const engines = require('./package.json').engines;
-const indexJsRestart = 'indexjs.restart';
 const fs = require('fs');
+const os = require('os');
 const path = require('path');
 const {exec} = require('child_process');
 const rimraf = require('rimraf');
@@ -15,12 +13,12 @@ let stopping = false;
 const hashFile = path.join(__dirname, 'dist', '.hash');
 
 async function restart() {
-    await stop(indexJsRestart);
+    await stop(true);
     await start();
 }
 
-async function exit(code, reason) {
-    if (reason !== indexJsRestart) {
+async function exit(code, restart) {
+    if (!restart) {
         process.exit(code);
     }
 }
@@ -44,7 +42,15 @@ async function build(reason) {
     return new Promise((resolve, reject) => {
         process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
         rimraf.sync('dist');
-        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+        const env = {...process.env};
+        const _600mb = 629145600;
+        if (_600mb > os.totalmem() && !env.NODE_OPTIONS) {
+            // Prevent OOM on tsc compile for system with low memory
+            // https://github.com/Koenkk/zigbee2mqtt/issues/12034
+            env.NODE_OPTIONS = '--max_old_space_size=256';
+        }
+
+        exec('npm run build', {env, cwd: __dirname}, async (err, stdout, stderr) => {
             if (err) {
                 process.stdout.write(', failed\n');
                 if (err.code === 134) {
@@ -101,21 +107,27 @@ async function start() {
     await controller.start();
 }
 
-async function stop(reason=null) {
-    await controller.stop(reason);
+async function stop(restart) {
+    await controller.stop(restart);
 }
 
 async function handleQuit() {
     if (!stopping && controller) {
         stopping = true;
-        await stop();
+        await stop(false);
     }
 }
 
-if (process.argv.length === 3 && process.argv[2] === 'writehash') {
-    writeHash();
+if (require.main === module || require.main.filename.endsWith(path.sep + 'cli.js')) {
+    if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+        writeHash();
+    } else {
+        process.on('SIGINT', handleQuit);
+        process.on('SIGTERM', handleQuit);
+        start();
+    }
 } else {
     process.on('SIGINT', handleQuit);
     process.on('SIGTERM', handleQuit);
-    start();
+    module.exports = {start};
 }

lib/controller.ts

@@ -39,24 +39,25 @@ const AllExtensions = [
 type ExtensionArgs = [Zigbee, MQTT, State, PublishEntityState, EventBus,
     (enable: boolean, name: string) => Promise<void>, () => void, (extension: Extension) => Promise<void>];
 
-class Controller {
+export class Controller {
     private eventBus: EventBus;
     private zigbee: Zigbee;
     private state: State;
     private mqtt: MQTT;
     private restartCallback: () => void;
-    private exitCallback: (code: number) => void;
+    private exitCallback: (code: number, restart: boolean) => void;
     private extensions: Extension[];
     private extensionArgs: ExtensionArgs;
 
-    constructor(restartCallback: () => void, exitCallback: (code: number) => void) {
+    constructor(restartCallback: () => void, exitCallback: (code: number, restart: boolean) => void) {
+        logger.init();
         this.eventBus = new EventBus( /* istanbul ignore next */ (error) => {
             logger.error(`Error: ${error.message}`);
             logger.debug(error.stack);
         });
         this.zigbee = new Zigbee(this.eventBus);
         this.mqtt = new MQTT(this.eventBus);
-        this.state = new State(this.eventBus);
+        this.state = new State(this.eventBus, this.zigbee);
         this.restartCallback = restartCallback;
         this.exitCallback = exitCallback;
 
@@ -111,6 +112,7 @@ class Controller {
         if (startResult === 'reset') {
             settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
             settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['advanced', 'legacy_availability_payload'], false);
             settings.set(['device_options', 'legacy'], false);
             this.enableDisableExtension(false, 'BridgeLegacy');
         }
@@ -142,8 +144,7 @@ class Controller {
         try {
             await this.mqtt.connect();
         } catch (error) {
-            logger.error(`MQTT failed to connect: ${error.message}`);
-            logger.error('Exiting...');
+            logger.error(`MQTT failed to connect, exiting...`);
             await this.zigbee.stop();
             await this.exit(1);
         }
@@ -153,15 +154,17 @@ class Controller {
 
         // Send all cached states.
         if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
-            for (const device of devices) {
-                if (this.state.exists(device)) {
-                    this.publishEntityState(device, this.state.get(device));
+            for (const entity of [...devices, ...this.zigbee.groups()]) {
+                if (this.state.exists(entity)) {
+                    this.publishEntityState(entity, this.state.get(entity), 'publishCached');
                 }
             }
         }
 
         this.eventBus.onLastSeenChanged(this,
             (data) => utils.publishLastSeen(data, settings.get(), false, this.publishEntityState));
+
+        logger.info(`Zigbee2MQTT started!`);
     }
 
     @bind async enableDisableExtension(enable: boolean, name: string): Promise<void> {
@@ -185,7 +188,7 @@ class Controller {
         await this.callExtensions('start', [extension]);
     }
 
-    async stop(): Promise<void> {
+    async stop(restart = false): Promise<void> {
         // Call extensions
         await this.callExtensions('stop', this.extensions);
         this.eventBus.removeListeners(this);
@@ -197,16 +200,16 @@ class Controller {
         try {
             await this.zigbee.stop();
             logger.info('Stopped Zigbee2MQTT');
-            await this.exit(0);
+            await this.exit(0, restart);
         } catch (error) {
             logger.error('Failed to stop Zigbee2MQTT');
-            await this.exit(1);
+            await this.exit(1, restart);
         }
     }
 
-    async exit(code: number): Promise<void> {
+    async exit(code: number, restart = false): Promise<void> {
         await logger.end();
-        this.exitCallback(code);
+        this.exitCallback(code, restart);
     }
 
     @bind async onZigbeeAdapterDisconnected(): Promise<void> {
@@ -263,10 +266,8 @@ class Controller {
             extension.adjustMessageBeforePublish?.(entity, message);
         }
 
-        // filter mqtt message attributes
-        if (entity.options.filtered_attributes) {
-            entity.options.filtered_attributes.forEach((a) => delete message[a]);
-        }
+        // Filter mqtt message attributes
+        utils.filterProperties(entity.options.filtered_attributes, message);
 
         if (Object.entries(message).length) {
             const output = settings.get().advanced.output;

lib/eventBus.ts

@@ -27,6 +27,13 @@ export default class EventBus {
         this.on('permitJoinChanged', callback, key);
     }
 
+    public emitPublishAvailability(): void {
+        this.emitter.emit('publishAvailability');
+    }
+    public onPublishAvailability(key: ListenerKey, callback: () => void): void {
+        this.on('publishAvailability', callback, key);
+    }
+
     public emitEntityRenamed(data: eventdata.EntityRenamed): void {
         this.emitter.emit('deviceRenamed', data);
     }

lib/extension/availability.ts

@@ -5,6 +5,12 @@ import * as settings from '../util/settings';
 import debounce from 'debounce';
 import bind from 'bind-decorator';
 
+const retrieveOnReconnect = [
+    {keys: ['state']},
+    {keys: ['brightness'], condition: (state: KeyValue): boolean => state.state === 'ON'},
+    {keys: ['color', 'color_temp'], condition: (state: KeyValue): boolean => state.state === 'ON'},
+];
+
 export default class Availability extends Extension {
     private timers: {[s: string]: NodeJS.Timeout} = {};
     private availabilityCache: {[s: string]: boolean} = {};
@@ -18,7 +24,8 @@ export default class Availability extends Extension {
         }
 
         const key = this.isActiveDevice(device) ? 'active' : 'passive';
-        const value = settings.get().availability[key]?.timeout;
+        let value = settings.get().availability?.[key]?.timeout;
+        if (value == null) value = key == 'active' ? 10 : 1500;
         return utils.minutes(value);
     }
 
@@ -27,9 +34,14 @@ export default class Availability extends Extension {
             device.zh.powerSource === 'Mains (single phase)';
     }
 
-    private isAvailable(device: Device): boolean {
-        const ago = Date.now() - device.zh.lastSeen;
-        return ago < this.getTimeout(device);
+    private isAvailable(entity: Device | Group): boolean {
+        if (entity.isDevice()) {
+            const ago = Date.now() - entity.zh.lastSeen;
+            return ago < this.getTimeout(entity);
+        } else {
+            return entity.membersDevices().length === 0 ||
+                entity.membersDevices().map((d) => this.availabilityCache[d.ieeeAddr]).includes(true);
+        }
     }
 
     private resetTimer(device: Device): void {
@@ -92,7 +104,7 @@ export default class Availability extends Extension {
 
     override async start(): Promise<void> {
         this.eventBus.onEntityRenamed(this, (data) => {
-            if (data.entity.isDevice() && utils.isAvailabilityEnabledForDevice(data.entity, settings.get())) {
+            if (utils.isAvailabilityEnabledForEntity(data.entity, settings.get())) {
                 this.mqtt.publish(`${data.from}/availability`, null, {retain: true, qos: 0});
                 this.publishAvailability(data.entity, false, true);
             }
@@ -102,53 +114,67 @@ export default class Availability extends Extension {
         this.eventBus.onDeviceLeave(this, (data) => clearTimeout(this.timers[data.ieeeAddr]));
         this.eventBus.onDeviceAnnounce(this, (data) => this.retrieveState(data.device));
         this.eventBus.onLastSeenChanged(this, this.onLastSeenChanged);
+        this.eventBus.onPublishAvailability(this, this.publishAvailabilityForAllEntities);
+        this.eventBus.onGroupMembersChanged(this, (data) => this.publishAvailability(data.group, false));
+        this.publishAvailabilityForAllEntities();
+    }
 
-        for (const device of this.zigbee.devices(false)) {
-            if (utils.isAvailabilityEnabledForDevice(device, settings.get())) {
+    @bind private publishAvailabilityForAllEntities(): void {
+        for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+            if (utils.isAvailabilityEnabledForEntity(entity, settings.get())) {
                 // Publish initial availablility
-                this.publishAvailability(device, true);
+                this.publishAvailability(entity, true, false, true);
 
-                this.resetTimer(device);
+                if (entity.isDevice()) {
+                    this.resetTimer(entity);
 
-                // If an active device is initially unavailable, ping it.
-                if (this.isActiveDevice(device) && !this.isAvailable(device)) {
-                    this.addToPingQueue(device);
+                    // If an active device is initially unavailable, ping it.
+                    if (this.isActiveDevice(entity) && !this.isAvailable(entity)) {
+                        this.addToPingQueue(entity);
+                    }
                 }
             }
         }
     }
 
-    private publishAvailability(device: Device, logLastSeen: boolean, forcePublish=false): void {
-        if (logLastSeen) {
-            const ago = Date.now() - device.zh.lastSeen;
-            if (this.isActiveDevice(device)) {
-                logger.debug(`Active device '${device.name}' was last seen ` +
+    private publishAvailability(entity: Device | Group, logLastSeen: boolean,
+        forcePublish=false, skipGroups=false): void {
+        if (logLastSeen && entity.isDevice()) {
+            const ago = Date.now() - entity.zh.lastSeen;
+            if (this.isActiveDevice(entity)) {
+                logger.debug(`Active device '${entity.name}' was last seen ` +
                     `'${(ago / utils.minutes(1)).toFixed(2)}' minutes ago.`);
             } else {
                 logger.debug(
-                    `Passive device '${device.name}' was last seen '${(ago / utils.hours(1)).toFixed(2)}' hours ago.`);
+                    `Passive device '${entity.name}' was last seen '${(ago / utils.hours(1)).toFixed(2)}' hours ago.`);
             }
         }
 
-        const available = this.isAvailable(device);
-        if (!forcePublish && this.availabilityCache[device.ieeeAddr] == available) {
+        const available = this.isAvailable(entity);
+        if (!forcePublish && this.availabilityCache[entity.ID] == available) {
             return;
         }
 
-        if (device.ieeeAddr in this.availabilityCache && available &&
-            this.availabilityCache[device.ieeeAddr] === false) {
-            logger.debug(`Device '${device.name}' reconnected`);
-            this.retrieveState(device);
+        if (entity.isDevice() && entity.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[entity.ieeeAddr] === false) {
+            logger.debug(`Device '${entity.name}' reconnected`);
+            this.retrieveState(entity);
         }
 
-        const topic = `${device.name}/availability`;
-        const payload = available ? 'online' : 'offline';
-        this.availabilityCache[device.ieeeAddr] = available;
+        const topic = `${entity.name}/availability`;
+        const payload = utils.availabilityPayload(available ? 'online' : 'offline', settings.get());
+        this.availabilityCache[entity.ID] = available;
         this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+
+        if (!skipGroups && entity.isDevice()) {
+            this.zigbee.groups().filter((g) => g.hasMember(entity))
+                .filter((g) => utils.isAvailabilityEnabledForEntity(g, settings.get()))
+                .forEach((g) => this.publishAvailability(g, false, forcePublish));
+        }
     }
 
     @bind private onLastSeenChanged(data: eventdata.LastSeenChanged): void {
-        if (utils.isAvailabilityEnabledForDevice(data.device, settings.get())) {
+        if (utils.isAvailabilityEnabledForEntity(data.device, settings.get())) {
             // Remove from ping queue, not necessary anymore since we know the device is online.
             this.removeFromPingQueue(data.device);
             this.resetTimer(data.device);
@@ -167,17 +193,18 @@ export default class Availability extends Extension {
          * device can send multiple times after each other.
          */
         if (device.definition && !device.zh.interviewing && !this.retrieveStateDebouncers[device.ieeeAddr]) {
-            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(() => {
+            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(async () => {
                 logger.debug(`Retrieving state of '${device.name}' after reconnect`);
                 // Color and color temperature converters do both, only needs to be called once.
-                const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
-                for (const keys of keySet) {
-                    const converter = device.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
-                    converter?.convertGet?.(device.endpoint(), keys[0],
-                        {message: this.state.get(device) || {}, mapped: device.definition})
+                for (const item of retrieveOnReconnect) {
+                    if (item.condition && this.state.get(device) && !item.condition(this.state.get(device))) continue;
+                    const converter = device.definition.toZigbee.find((c) => c.key.find((k) => item.keys.includes(k)));
+                    await converter?.convertGet?.(device.endpoint(), item.keys[0],
+                        {message: this.state.get(device), mapped: device.definition})
                         .catch((e) => {
                             logger.error(`Failed to read state of '${device.name}' after reconnect (${e.message})`);
                         });
+                    await utils.sleep(500);
                 }
             }, utils.seconds(2));
         }

lib/extension/bind.ts

@@ -13,7 +13,8 @@ const legacyApi = settings.get().advanced.legacy_api;
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
-    'hvacThermostat', 'msTemperatureMeasurement'];
+    'hvacThermostat', 'msIlluminanceMeasurement', 'msTemperatureMeasurement', 'msRelativeHumidity',
+    'msSoilMoisture', 'msCO2'];
 
 // See zigbee-herdsman-converters
 const defaultBindGroup = {type: 'group_number', ID: 901, name: 'default_bind_group'};
@@ -67,7 +68,8 @@ const reportClusters: {[s: string]:
 type PollOnMessage = {
     cluster: {[s: string]: {type: string, data: KeyValue}[]}
     read: {cluster: string, attributes: string[], attributesForEndpoint?: (endpoint: zh.Endpoint) => Promise<string[]>}
-    manufacturerIDs: number[]
+    manufacturerIDs: number[],
+    manufacturerNames: string [],
 }[];
 
 const pollOnMessage: PollOnMessage = [
@@ -101,6 +103,10 @@ const pollOnMessage: PollOnMessage = [
             zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
             zigbeeHersdman.Zcl.ManufacturerCode.TELINK,
         ],
+        manufacturerNames: [
+            'GLEDOPTO',
+            'Trust International B.V.\u0000',
+        ],
     },
     {
         cluster: {
@@ -132,6 +138,10 @@ const pollOnMessage: PollOnMessage = [
             zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
             zigbeeHersdman.Zcl.ManufacturerCode.TELINK,
         ],
+        manufacturerNames: [
+            'GLEDOPTO',
+            'Trust International B.V.\u0000',
+        ],
     },
     {
         cluster: {
@@ -159,6 +169,10 @@ const pollOnMessage: PollOnMessage = [
             zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
             zigbeeHersdman.Zcl.ManufacturerCode.TELINK,
         ],
+        manufacturerNames: [
+            'GLEDOPTO',
+            'Trust International B.V.\u0000',
+        ],
     },
 ];
 
@@ -455,7 +469,8 @@ export default class Bind extends Extension {
 
             for (const endpoint of toPoll) {
                 for (const poll of polls) {
-                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID) ||
+                    if ((!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID) &&
+                        !poll.manufacturerNames.includes(endpoint.getDevice().manufacturerName)) ||
                         !endpoint.supportsInputCluster(poll.read.cluster)) {
                         continue;
                     }
@@ -469,7 +484,12 @@ export default class Bind extends Extension {
                     const key = `${endpoint.getDevice().ieeeAddr}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
                     if (!this.pollDebouncers[key]) {
                         this.pollDebouncers[key] = debounce(async () => {
-                            await endpoint.read(poll.read.cluster, readAttrs);
+                            try {
+                                await endpoint.read(poll.read.cluster, readAttrs);
+                            } catch (error) {
+                                logger.error(`Failed to poll ${readAttrs} from ` +
+                                    `${this.zigbee.resolveEntity(endpoint.getDevice()).name}`);
+                            }
                         }, 1000);
                     }
 

lib/extension/bridge.ts

@@ -6,10 +6,12 @@ import Transport from 'winston-transport';
 import bind from 'bind-decorator';
 import stringify from 'json-stable-stringify-without-jsonify';
 import objectAssignDeep from 'object-assign-deep';
-import {detailedDiff} from 'deep-object-diff';
 import Extension from './extension';
 import Device from '../model/device';
 import Group from '../model/group';
+import data from '../util/data';
+import JSZip from 'jszip';
+import fs from 'fs';
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
@@ -38,8 +40,10 @@ export default class Bridge extends Extension {
             'group/rename': this.groupRename,
             'permit_join': this.permitJoin,
             'restart': this.restart,
+            'backup': this.backup,
             'touchlink/factory_reset': this.touchlinkFactoryReset,
             'touchlink/identify': this.touchlinkIdentify,
+            'install_code/add': this.installCodeAdd,
             'touchlink/scan': this.touchlinkScan,
             'health_check': this.healthCheck,
             'options': this.bridgeOptions,
@@ -143,35 +147,7 @@ export default class Bridge extends Extension {
             throw new Error(`Invalid payload`);
         }
 
-        const diff: KeyValue = detailedDiff(settings.get(), message.options);
-
-        // Remove any settings that are in the deleted.diff but not in the passed options
-        const cleanupDeleted = (options: KeyValue, deleted: KeyValue): void => {
-            for (const key of Object.keys(deleted)) {
-                if (!(key in options)) {
-                    delete deleted[key];
-                } else if (!Array.isArray(options[key])) {
-                    cleanupDeleted(options[key], deleted[key]);
-                }
-            }
-        };
-        cleanupDeleted(message.options, diff.deleted);
-
-        const newSettings = objectAssignDeep({}, diff.added, diff.updated, diff.deleted);
-
-        // deep-object-diff converts arrays to objects, set original array back here
-        const convertBackArray = (before: KeyValue, after: KeyValue): void => {
-            for (const [key, afterValue] of Object.entries(after)) {
-                const beforeValue = before[key];
-                if (Array.isArray(beforeValue)) {
-                    after[key] = beforeValue;
-                } else if (afterValue && typeof beforeValue === 'object') {
-                    convertBackArray(beforeValue, afterValue);
-                }
-            }
-        };
-        convertBackArray(message.options, newSettings);
-
+        const newSettings = utils.computeSettingsToChange(settings.get(), message.options);
         const restartRequired = settings.apply(newSettings);
         if (restartRequired) this.restartRequired = true;
 
@@ -233,6 +209,28 @@ export default class Bridge extends Extension {
         return utils.getResponse(message, {}, null);
     }
 
+    @bind async backup(message: string | KeyValue): Promise<MQTTResponse> {
+        await this.zigbee.backup();
+        const dataPath = data.getPath();
+        const files = utils.getAllFiles(dataPath).map((f) => [f, f.substring(dataPath.length + 1)])
+            .filter((f) => !f[1].startsWith('log'));
+        const zip = new JSZip();
+        files.forEach((f) => zip.file(f[1], fs.readFileSync(f[0])));
+        const base64Zip = await zip.generateAsync({type: 'base64'});
+        return utils.getResponse(message, {zip: base64Zip}, null);
+    }
+
+    @bind async installCodeAdd(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        const value = typeof message === 'object' ? message.value : message;
+        await this.zigbee.addInstallCode(value);
+        logger.info('Successfully added new install code');
+        return utils.getResponse(message, {value}, null);
+    }
+
     @bind async permitJoin(message: KeyValue | string): Promise<MQTTResponse> {
         if (typeof message === 'object' && !message.hasOwnProperty('value')) {
             throw new Error('Invalid payload');
@@ -392,15 +390,23 @@ export default class Bridge extends Extension {
 
         const ID = message.id;
         const entity = this.getEntity(entityType, ID);
+        const currentOptions = entityType === 'device' ? settings.get().devices[entity.ID] :
+            settings.get().groups[entity.ID];
+        const options = utils.computeSettingsToChange(currentOptions, message.options);
         const oldOptions = objectAssignDeep({}, cleanup(entity.options));
-        settings.changeEntityOptions(ID, message.options);
+        const restartRequired = settings.changeEntityOptions(ID, options);
+        if (restartRequired) this.restartRequired = true;
         const newOptions = cleanup(entity.options);
         await this.publishInfo();
 
         logger.info(`Changed config for ${entityType} ${ID}`);
 
         this.eventBus.emitEntityOptionsChanged({from: oldOptions, to: newOptions, entity});
-        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+        return utils.getResponse(
+            message,
+            {from: oldOptions, to: newOptions, id: ID, restart_required: this.restartRequired},
+            null,
+        );
     }
 
     @bind async deviceConfigureReporting(message: string | KeyValue): Promise<MQTTResponse> {
@@ -419,7 +425,7 @@ export default class Bridge extends Extension {
         await endpoint.configureReporting(message.cluster, [{
             attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
             maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
-        }]);
+        }], message.options);
 
         this.publishDevices();
 
@@ -650,6 +656,8 @@ export default class Bridge extends Extension {
                 network_address: device.zh.networkAddress,
                 supported: !!device.definition,
                 friendly_name: device.name,
+                disabled: !!device.options.disabled,
+                description: device.options.description,
                 definition: this.getDefinitionPayload(device),
                 power_source: device.zh.powerSource,
                 software_build_id: device.zh.softwareBuildID,
@@ -671,6 +679,7 @@ export default class Bridge extends Extension {
             return {
                 id: g.ID,
                 friendly_name: g.ID === 901 ? 'default_bind_group' : g.name,
+                description: g.options.description,
                 scenes: this.getScenes(g.zh),
                 members: g.zh.members.map((e) => {
                     return {ieee_address: e.getDevice().ieeeAddr, endpoint: e.ID};

lib/extension/configure.ts

@@ -64,9 +64,11 @@ export default class Configure extends Extension {
     }
 
     override async start(): Promise<void> {
-        for (const device of this.zigbee.devices(false)) {
-            await this.configure(device, 'started');
-        }
+        setImmediate(async () => {
+            for (const device of this.zigbee.devices(false)) {
+                await this.configure(device, 'started');
+            }
+        });
 
         this.eventBus.onDeviceJoined(this, (data) => {
             if (data.device.zh.meta.hasOwnProperty('configured')) {
@@ -85,7 +87,7 @@ export default class Configure extends Extension {
     private async configure(device: Device, event: 'started' | 'zigbee_event' | 'reporting_disabled' | 'mqtt_message',
         force=false, thowError=false): Promise<void> {
         if (!force) {
-            if (!device.definition?.configure || device.zh.interviewing) {
+            if (device.options.disabled || !device.definition?.configure || !device.zh.interviewCompleted) {
                 return;
             }
 

lib/extension/externalExtension.ts

@@ -56,7 +56,7 @@ export default class ExternalExtension extends Extension {
 
     @bind private async saveExtension(message: KeyValue): Promise<MQTTResponse> {
         const {name, code} = message;
-        const ModuleConstructor = utils.loadModuleFromText(code) as typeof Extension;
+        const ModuleConstructor = utils.loadModuleFromText(code, name) as typeof Extension;
         await this.loadExtension(ModuleConstructor);
         const basePath = this.getExtensionsBasePath();
         /* istanbul ignore else */
@@ -95,7 +95,7 @@ export default class ExternalExtension extends Extension {
     private loadUserDefinedExtensions(): void {
         const extensions = this.getListOfUserDefinedExtensions();
         extensions
-            .map(({code}) => utils.loadModuleFromText(code))
+            .map(({code, name}) => utils.loadModuleFromText(code, name))
             .map(this.loadExtension);
     }
 

lib/extension/frontend.ts

@@ -1,4 +1,5 @@
 import http from 'http';
+import https from 'https';
 import gzipStatic, {RequestHandler} from 'connect-gzip-static';
 import finalhandler from 'finalhandler';
 import logger from '../util/logger';
@@ -6,6 +7,7 @@ import frontend from 'zigbee2mqtt-frontend';
 import WebSocket from 'ws';
 import net from 'net';
 import url from 'url';
+import fs from 'fs';
 import * as settings from '../util/settings';
 import utils from '../util/utils';
 import stringify from 'json-stable-stringify-without-jsonify';
@@ -19,6 +21,8 @@ export default class Frontend extends Extension {
     private mqttBaseTopic = settings.get().mqtt.base_topic;
     private host = settings.get().frontend.host;
     private port = settings.get().frontend.port;
+    private sslCert = settings.get().frontend.ssl_cert;
+    private sslKey = settings.get().frontend.ssl_key;
     private authToken = settings.get().frontend.auth_token;
     private retainedMessages = new Map();
     private server: http.Server;
@@ -33,8 +37,28 @@ export default class Frontend extends Extension {
         this.eventBus.onMQTTMessagePublished(this, this.onMQTTPublishMessage);
     }
 
+    private isHttpsConfigured():boolean {
+        if (this.sslCert && this.sslKey) {
+            if (!fs.existsSync(this.sslCert) || !fs.existsSync(this.sslKey)) {
+                logger.error(`defined ssl_cert '${this.sslCert}' or ssl_key '${this.sslKey}' file path does not exists, server won't be secured.`); /* eslint-disable-line max-len */
+                return false;
+            }
+            return true;
+        }
+        return false;
+    }
+
+
     override async start(): Promise<void> {
-        this.server = http.createServer(this.onRequest);
+        if (this.isHttpsConfigured()) {
+            const serverOptions = {
+                key: fs.readFileSync(this.sslKey),
+                cert: fs.readFileSync(this.sslCert)};
+            this.server = https.createServer(serverOptions, this.onRequest);
+        } else {
+            this.server = http.createServer(this.onRequest);
+        }
+
         this.server.on('upgrade', this.onUpgrade);
 
         /* istanbul ignore next */
@@ -100,11 +124,7 @@ export default class Frontend extends Extension {
         }
 
         for (const device of this.zigbee.devices(false)) {
-            let payload: KeyValue = {};
-            if (this.state.exists(device)) {
-                payload = {...payload, ...this.state.get(device)};
-            }
-
+            const payload = this.state.get(device);
             const lastSeen = settings.get().advanced.last_seen;
             /* istanbul ignore if */
             if (lastSeen !== 'disable') {

lib/extension/groups.ts

@@ -101,7 +101,7 @@ export default class Groups extends Extension {
 
     @bind async onStateChange(data: eventdata.StateChange): Promise<void> {
         const reason = 'groupOptimistic';
-        if (data.reason === reason) {
+        if (data.reason === reason || data.reason === 'publishCached') {
             return;
         }
 
@@ -129,11 +129,10 @@ export default class Groups extends Extension {
             if (entity instanceof Device) {
                 for (const group of groups) {
                     if (group.zh.hasMember(entity.endpoint(endpointName)) &&
-                        !equals(this.lastOptimisticState[group.ID], payload)) {
-                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(group)) {
-                            this.lastOptimisticState[group.ID] = payload;
-                            await this.publishEntityState(group, payload, reason);
-                        }
+                        !equals(this.lastOptimisticState[group.ID], payload) &&
+                        this.shouldPublishPayloadForGroup(group, payload)) {
+                        this.lastOptimisticState[group.ID] = payload;
+                        await this.publishEntityState(group, payload, reason);
                     }
                 }
             } else {
@@ -143,6 +142,7 @@ export default class Groups extends Extension {
                 const groupsToPublish: Set<Group> = new Set();
                 for (const member of entity.zh.members) {
                     const device = this.zigbee.resolveEntity(member.getDevice()) as Device;
+                    if (device.options.disabled) continue;
                     const exposes = device.exposes();
                     const memberPayload: KeyValue = {};
                     Object.keys(payload).forEach((key) => {
@@ -161,10 +161,9 @@ export default class Groups extends Extension {
 
                     await this.publishEntityState(device, memberPayload, reason);
                     for (const zigbeeGroup of groups) {
-                        if (zigbeeGroup.zh.hasMember(member)) {
-                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
-                                groupsToPublish.add(zigbeeGroup);
-                            }
+                        if (zigbeeGroup.zh.hasMember(member) &&
+                            this.shouldPublishPayloadForGroup(zigbeeGroup, payload)) {
+                            groupsToPublish.add(zigbeeGroup);
                         }
                     }
                 }
@@ -176,12 +175,19 @@ export default class Groups extends Extension {
         }
     }
 
+    private shouldPublishPayloadForGroup(group: Group, payload: KeyValue): boolean {
+        if (group.options.off_state === 'last_member_state') return true;
+        if (!payload || payload.state !== 'OFF') return true;
+        if (this.areAllMembersOff(group)) return true;
+        return false;
+    }
+
     private areAllMembersOff(group: Group): boolean {
         for (const member of group.zh.members) {
             const device = this.zigbee.resolveEntity(member.getDevice());
             if (this.state.exists(device)) {
                 const state = this.state.get(device);
-                if (state && state.state === 'ON') {
+                if (state.state === 'ON') {
                     return false;
                 }
             }
@@ -287,6 +293,7 @@ export default class Groups extends Extension {
             groupKey, deviceKey, skipDisableReporting, resolvedEntityEndpoint,
         } = parsed;
         const message = utils.parseJSON(data.message, data.message);
+        let changedGroups: Group[] = [];
 
         const responseData: KeyValue = {device: deviceKey};
         if (groupKey) {
@@ -315,6 +322,7 @@ export default class Groups extends Extension {
                     logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
                     await resolvedEntityEndpoint.addToGroup(resolvedEntityGroup.zh);
                     settings.addDeviceToGroup(resolvedEntityGroup.ID.toString(), keys);
+                    changedGroups.push(resolvedEntityGroup);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
@@ -328,6 +336,7 @@ export default class Groups extends Extension {
                     logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
                     await resolvedEntityEndpoint.removeFromGroup(resolvedEntityGroup.zh);
                     settings.removeDeviceFromGroup(resolvedEntityGroup.ID.toString(), keys);
+                    changedGroups.push(resolvedEntityGroup);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
@@ -339,6 +348,7 @@ export default class Groups extends Extension {
                     }
                 } else { // remove_all
                     logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    changedGroups = this.zigbee.groups().filter((g) => g.zh.members.includes(resolvedEntityEndpoint));
                     await resolvedEntityEndpoint.removeFromAllGroups();
                     for (const settingsGroup of settings.getGroups()) {
                         settings.removeDeviceFromGroup(settingsGroup.ID.toString(), keys);
@@ -367,8 +377,10 @@ export default class Groups extends Extension {
         if (error) {
             logger.error(error);
         } else {
-            this.eventBus.emitGroupMembersChanged({
-                group: resolvedEntityGroup, action: type, endpoint: resolvedEntityEndpoint, skipDisableReporting});
+            for (const group of changedGroups) {
+                this.eventBus.emitGroupMembersChanged({
+                    group, action: type, endpoint: resolvedEntityEndpoint, skipDisableReporting});
+            }
         }
     }
 }

lib/extension/homeassistant.ts

@@ -95,6 +95,7 @@ export default class HomeAssistant extends Extension {
         this.eventBus.onDeviceJoined(this, this.onZigbeeEvent);
         this.eventBus.onDeviceInterview(this, this.onZigbeeEvent);
         this.eventBus.onDeviceMessage(this, this.onZigbeeEvent);
+        this.eventBus.onEntityOptionsChanged(this, (data) => this.discover(data.entity, true));
 
         this.mqtt.subscribe(this.statusTopic);
         this.mqtt.subscribe(defaultStatusTopic);
@@ -104,10 +105,13 @@ export default class HomeAssistant extends Extension {
         for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
             this.discover(entity, true);
         }
+
+        // Send availability messages, this is required if the legacy_availability_payload option has been changed.
+        this.eventBus.emitPublishAvailability();
     }
 
     private exposeToConfig(exposes: zhc.DefinitionExpose[], entityType: 'device' | 'group',
-        definition?: zhc.Definition, definitionExposes?: zhc.DefinitionExpose[]): DiscoveryEntry[] {
+        allExposes: zhc.DefinitionExpose[], definition?: zhc.Definition): DiscoveryEntry[] {
         // For groups an array of exposes (of the same type) is passed, this is to determine e.g. what features
         // to use for a bulb (e.g. color_xy/color_temp)
         assert(entityType === 'group' || exposes.length === 1, 'Multiple exposes for device not allowed');
@@ -167,10 +171,11 @@ export default class HomeAssistant extends Extension {
                 discoveryEntry.discovery_payload.min_mireds = min;
             }
 
-            const effect = definitionExposes?.find((e) => e.type === 'enum' && e.name === 'effect');
-            if (effect) {
+            const effects = utils.arrayUnique(utils.flatten(
+                allExposes.filter((e) => e.type === 'enum' && e.name === 'effect').map((e) => e.values)));
+            if (effects.length) {
                 discoveryEntry.discovery_payload.effect = true;
-                discoveryEntry.discovery_payload.effect_list = effect.values;
+                discoveryEntry.discovery_payload.effect_list = effects;
             }
 
             discoveryEntries.push(discoveryEntry);
@@ -278,21 +283,22 @@ export default class HomeAssistant extends Extension {
                 discoveryEntry.discovery_payload.fan_mode_state_topic = true;
             }
 
-            const preset = firstExpose.features.find((f) => f.name === 'preset');
-            if (preset) {
-                discoveryEntry.discovery_payload.hold_modes = preset.values;
-                discoveryEntry.discovery_payload.hold_command_topic = true;
-                discoveryEntry.discovery_payload.hold_state_template =
-                    `{{ value_json.${preset.property} }}`;
-                discoveryEntry.discovery_payload.hold_state_topic = true;
+            const swingMode = firstExpose.features.find((f) => f.name === 'swing_mode');
+            if (swingMode) {
+                discoveryEntry.discovery_payload.swing_modes = swingMode.values;
+                discoveryEntry.discovery_payload.swing_mode_command_topic = true;
+                discoveryEntry.discovery_payload.swing_mode_state_template =
+                    `{{ value_json.${swingMode.property} }}`;
+                discoveryEntry.discovery_payload.swing_mode_state_topic = true;
             }
 
-            const awayMode = firstExpose.features.find((f) => f.name === 'away_mode');
-            if (awayMode) {
-                discoveryEntry.discovery_payload.away_mode_command_topic = true;
-                discoveryEntry.discovery_payload.away_mode_state_topic = true;
-                discoveryEntry.discovery_payload.away_mode_state_template =
-                    `{{ value_json.${awayMode.property} }}`;
+            const preset = firstExpose.features.find((f) => f.name === 'preset');
+            if (preset) {
+                discoveryEntry.discovery_payload.preset_modes = preset.values;
+                discoveryEntry.discovery_payload.preset_mode_command_topic = 'preset';
+                discoveryEntry.discovery_payload.preset_mode_value_template =
+                    `{{ value_json.${preset.property} }}`;
+                discoveryEntry.discovery_payload.preset_mode_state_topic = true;
             }
 
             const tempCalibration = firstExpose.features.find((f) => f.name === 'local_temperature_calibration');
@@ -306,6 +312,7 @@ export default class HomeAssistant extends Extension {
                         command_topic: true,
                         command_topic_prefix: endpoint,
                         command_topic_postfix: tempCalibration.property,
+                        device_class: 'temperature',
                         entity_category: 'config',
                         icon: 'mdi:math-compass',
                         ...(tempCalibration.unit && {unit_of_measurement: tempCalibration.unit}),
@@ -314,6 +321,9 @@ export default class HomeAssistant extends Extension {
 
                 if (tempCalibration.value_min != null) discoveryEntry.discovery_payload.min = tempCalibration.value_min;
                 if (tempCalibration.value_max != null) discoveryEntry.discovery_payload.max = tempCalibration.value_max;
+                if (tempCalibration.value_step != null) {
+                    discoveryEntry.discovery_payload.step = tempCalibration.value_step;
+                }
                 discoveryEntries.push(discoveryEntry);
             }
 
@@ -374,26 +384,56 @@ export default class HomeAssistant extends Extension {
 
             discoveryEntries.push(discoveryEntry);
         } else if (firstExpose.type === 'cover') {
-            const position = exposes.find((expose) => expose.features.find((e) => e.name === 'position'));
-            const tilt = exposes.find((expose) => expose.features.find((e) => e.name === 'tilt'));
+            const state = exposes.find((expose) => expose.features.find((e) => e.name === 'state'))
+                ?.features.find((f) => f.name === 'state');
+            const position = exposes.find((expose) => expose.features.find((e) => e.name === 'position'))
+                ?.features.find((f) => f.name === 'position');
+            const tilt = exposes.find((expose) => expose.features.find((e) => e.name === 'tilt'))
+                ?.features.find((f) => f.name === 'tilt');
+            const motorState = allExposes?.find((e) => e.type === 'enum' && e.name === 'motor_state' &&
+                e.access === ACCESS_STATE);
+            const running = allExposes?.find((e) => e.type === 'binary' && e.name === 'running');
 
             const discoveryEntry: DiscoveryEntry = {
                 type: 'cover',
-                mockProperties: [],
+                mockProperties: [{property: state.property, value: null}],
                 object_id: endpoint ? `cover_${endpoint}` : 'cover',
                 discovery_payload: {
                     command_topic_prefix: endpoint,
+                    command_topic: true,
+                    state_topic: true,
+                    state_topic_postfix: endpoint,
                 },
             };
 
-            // For covers only supporting tilt don't discover the command/state_topic, otherwise
-            // HA does not correctly reflect the state
-            // - https://github.com/home-assistant/core/issues/51793
-            // - https://github.com/Koenkk/zigbee-herdsman-converters/pull/2663
-            if (!tilt || (tilt && position)) {
-                discoveryEntry.discovery_payload.command_topic = true;
-                discoveryEntry.discovery_payload.state_topic = !position;
-                discoveryEntry.discovery_payload.command_topic_prefix = endpoint;
+            // For curtains that have `motor_state` lookup a possible state names and make this
+            // available for discovery. If the curtains only support the `running` value,
+            // then we use it anyway. The movement direction is calculated (assumed) in this case.
+            if (motorState) {
+                const openingLookup = ['opening', 'open', 'forward', 'up', 'rising'];
+                const closingLookup = ['closing', 'close', 'backward', 'back', 'reverse', 'down', 'declining'];
+                const stoppedLookup = ['stopped', 'stop', 'pause', 'paused'];
+
+                const openingState = motorState.values.find((s) => openingLookup.includes(s.toLowerCase()));
+                const closingState = motorState.values.find((s) => closingLookup.includes(s.toLowerCase()));
+                const stoppedState = motorState.values.find((s) => stoppedLookup.includes(s.toLowerCase()));
+
+                if (openingState && closingState && stoppedState) {
+                    discoveryEntry.discovery_payload.state_opening = openingState;
+                    discoveryEntry.discovery_payload.state_closing = closingState;
+                    discoveryEntry.discovery_payload.state_stopped = stoppedState;
+                    discoveryEntry.discovery_payload.value_template = `{% if not value_json.${motorState.property} %}` +
+                        ` ${stoppedState} {% else %} {{ value_json.${motorState.property} }} {% endif %}`;
+                }
+            } else if (running) {
+                discoveryEntry.discovery_payload.value_template = `{% if not value_json.${running.property} %} ` +
+                    `stopped {% else %} {% if value_json.${position.property} > 0 %} closing {% else %} ` +
+                    `opening {% endif %} {% endif %}`;
+            } else {
+                discoveryEntry.discovery_payload.value_template =
+                    `{{ value_json.${featurePropertyWithoutEndpoint(state)} }}`,
+                discoveryEntry.discovery_payload.state_open = 'OPEN';
+                discoveryEntry.discovery_payload.state_closed = 'CLOSE';
             }
 
             if (!position && !tilt) {
@@ -401,21 +441,19 @@ export default class HomeAssistant extends Extension {
             }
 
             if (position) {
-                const p = position.features.find((f) => f.name === 'position');
                 discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
-                    position_template: `{{ value_json.${getProperty(p)} }}`,
-                    set_position_template: `{ "${getProperty(p)}": {{ position }} }`,
+                    position_template: `{{ value_json.${featurePropertyWithoutEndpoint(position)} }}`,
+                    set_position_template: `{ "${getProperty(position)}": {{ position }} }`,
                     set_position_topic: true,
                     position_topic: true,
                 };
             }
 
             if (tilt) {
-                const t = tilt.features.find((f) => f.name === 'tilt');
                 discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
                     tilt_command_topic: true,
                     tilt_status_topic: true,
-                    tilt_status_template: `{{ value_json.${getProperty(t)} }}`,
+                    tilt_status_template: `{{ value_json.${featurePropertyWithoutEndpoint(tilt)} }}`,
                 };
             }
 
@@ -477,7 +515,7 @@ export default class HomeAssistant extends Extension {
                 discoveryEntry.discovery_payload.speed_range_max = speeds.length - 1;
                 assert(presets.length !== 0);
                 discoveryEntry.discovery_payload.preset_mode_state_topic = true;
-                discoveryEntry.discovery_payload.preset_mode_command_topic = true;
+                discoveryEntry.discovery_payload.preset_mode_command_topic = 'fan_mode';
                 discoveryEntry.discovery_payload.preset_mode_value_template =
                     `{{ value_json.${speed.property} if value_json.${speed.property} in [${presetList}]` +
                     ` else 'None' | default('None') }}`;
@@ -489,19 +527,23 @@ export default class HomeAssistant extends Extension {
             const lookup: {[s: string]: KeyValue}= {
                 battery_low: {entity_category: 'diagnostic', device_class: 'battery'},
                 button_lock: {entity_category: 'config', icon: 'mdi:lock'},
+                calibration: {entity_category: 'config', icon: 'mdi:progress-wrench'},
                 carbon_monoxide: {device_class: 'safety'},
                 card: {entity_category: 'config', icon: 'mdi:clipboard-check'},
                 child_lock: {entity_category: 'config', icon: 'mdi:account-lock'},
                 color_sync: {entity_category: 'config', icon: 'mdi:sync-circle'},
                 consumer_connected: {entity_category: 'diagnostic', device_class: 'connectivity'},
                 contact: {device_class: 'door'},
+                garage_door_contact: {device_class: 'garage_door', payload_on: false, payload_off: true},
                 eco_mode: {entity_category: 'config', icon: 'mdi:leaf'},
                 expose_pin: {entity_category: 'config', icon: 'mdi:pin'},
+                flip_indicator_light: {entity_category: 'config', icon: 'mdi:arrow-left-right'},
                 gas: {device_class: 'gas'},
                 invert_cover: {entity_category: 'config', icon: 'mdi:arrow-left-right'},
                 led_disabled_night: {entity_category: 'config', icon: 'mdi:led-off'},
                 led_indication: {entity_category: 'config', icon: 'mdi:led-on'},
                 legacy: {entity_category: 'config', icon: 'mdi:cog'},
+                motor_reversal: {entity_category: 'config', icon: 'mdi:arrow-left-right'},
                 moving: {device_class: 'moving'},
                 no_position_support: {entity_category: 'config', icon: 'mdi:minus-circle-outline'},
                 occupancy: {device_class: 'motion'},
@@ -510,9 +552,12 @@ export default class HomeAssistant extends Extension {
                 smoke: {device_class: 'smoke'},
                 sos: {device_class: 'safety'},
                 tamper: {device_class: 'tamper'},
+                temperature_scale: {entity_category: 'config', icon: 'mdi:temperature-celsius'},
                 test: {entity_category: 'diagnostic', icon: 'mdi:test-tube'},
+                valve_state: {device_class: 'opening'},
                 vibration: {device_class: 'vibration'},
                 water_leak: {device_class: 'moisture'},
+                window: {device_class: 'window'},
             };
 
             /**
@@ -557,6 +602,12 @@ export default class HomeAssistant extends Extension {
             }
         } else if (firstExpose.type === 'numeric') {
             const lookup: {[s: string]: KeyValue} = {
+                ac_frequency: {device_class: 'frequency', enabled_by_default: false, entity_category: 'diagnostic',
+                    state_class: 'measurement'},
+                alarm_humidity_max: {device_class: 'humidity', icon: 'mdi:water-plus'},
+                alarm_humidity_min: {device_class: 'humidity', icon: 'mdi:water-minus'},
+                alarm_temperature_max: {device_class: 'temperature', icon: 'mdi:thermometer-high'},
+                alarm_temperature_min: {device_class: 'temperature', icon: 'mdi:thermometer-low'},
                 angle: {icon: 'angle-acute'},
                 angle_axis: {icon: 'angle-acute'},
                 aqi: {device_class: 'aqi', state_class: 'measurement'},
@@ -564,9 +615,13 @@ export default class HomeAssistant extends Extension {
                 away_preset_days: {entity_category: 'config', icon: 'mdi:timer'},
                 away_preset_temperature: {entity_category: 'config', icon: 'mdi:thermometer'},
                 battery: {device_class: 'battery', entity_category: 'diagnostic', state_class: 'measurement'},
+                battery2: {device_class: 'battery', entity_category: 'diagnostic', state_class: 'measurement'},
                 battery_voltage: {device_class: 'voltage', entity_category: 'diagnostic', state_class: 'measurement'},
+                boost_heating_countdown: {device_class: 'duration'},
+                boost_heating_countdown_time_set: {entity_category: 'config', icon: 'mdi:timer'},
                 boost_time: {entity_category: 'config', icon: 'mdi:timer'},
-                calibration: {entity_category: 'config'},
+                calibration: {entity_category: 'config', icon: 'mdi:wrench-clock'},
+                calibration_time: {entity_category: 'config', icon: 'mdi:wrench-clock'},
                 co2: {device_class: 'carbon_dioxide', state_class: 'measurement'},
                 comfort_temperature: {entity_category: 'config', icon: 'mdi:thermometer'},
                 cpu_temperature: {
@@ -595,6 +650,7 @@ export default class HomeAssistant extends Extension {
                 device_temperature: {
                     device_class: 'temperature', entity_category: 'diagnostic', state_class: 'measurement',
                 },
+                duration: {entity_category: 'config', icon: 'mdi:timer'},
                 eco2: {device_class: 'carbon_dioxide', state_class: 'measurement'},
                 eco_temperature: {entity_category: 'config', icon: 'mdi:thermometer'},
                 energy: {device_class: 'energy', state_class: 'total_increasing'},
@@ -602,6 +658,10 @@ export default class HomeAssistant extends Extension {
                 gas_density: {icon: 'mdi:google-circles-communities', state_class: 'measurement'},
                 hcho: {icon: 'mdi:air-filter', state_class: 'measurement'},
                 humidity: {device_class: 'humidity', state_class: 'measurement'},
+                humidity_calibration: {entity_category: 'config', icon: 'mdi:wrench-clock'},
+                humidity_max: {entity_category: 'config', icon: 'mdi:water-percent'},
+                humidity_min: {entity_category: 'config', icon: 'mdi:water-percent'},
+                illuminance_calibration: {entity_category: 'config', icon: 'mdi:wrench-clock'},
                 illuminance_lux: {device_class: 'illuminance', state_class: 'measurement'},
                 illuminance: {device_class: 'illuminance', enabled_by_default: false, state_class: 'measurement'},
                 linkquality: {
@@ -611,18 +671,23 @@ export default class HomeAssistant extends Extension {
                     state_class: 'measurement',
                 },
                 local_temperature: {device_class: 'temperature', state_class: 'measurement'},
-                max_temperature: {entity_category: 'config', icon: 'mdi:thermometer'},
-                max_temperature_limit: {entity_category: 'config', icon: 'mdi:thermometer'},
-                min_temperature: {entity_category: 'config', icon: 'mdi:thermometer'},
+                max_temperature: {entity_category: 'config', icon: 'mdi:thermometer-high'},
+                max_temperature_limit: {entity_category: 'config', icon: 'mdi:thermometer-high'},
+                min_temperature_limit: {entity_category: 'config', icon: 'mdi:thermometer-low'},
+                min_temperature: {entity_category: 'config', icon: 'mdi:thermometer-low'},
                 measurement_poll_interval: {entity_category: 'config', icon: 'mdi:clock-out'},
                 occupancy_timeout: {entity_category: 'config', icon: 'mdi:timer'},
                 pm10: {device_class: 'pm10', state_class: 'measurement'},
                 pm25: {device_class: 'pm25', state_class: 'measurement'},
+                people: {state_class: 'measurement', icon: 'mdi:account-multiple'},
                 position: {icon: 'mdi:valve', state_class: 'measurement'},
                 power: {device_class: 'power', entity_category: 'diagnostic', state_class: 'measurement'},
+                power_factor: {device_class: 'power_factor', enabled_by_default: false,
+                    entity_category: 'diagnostic', state_class: 'measurement'},
                 precision: {entity_category: 'config', icon: 'mdi:decimal-comma-increase'},
                 pressure: {device_class: 'pressure', state_class: 'measurement'},
                 presence_timeout: {entity_category: 'config', icon: 'mdi:timer'},
+                reporting_time: {entity_category: 'config', icon: 'mdi:clock-time-one-outline'},
                 requested_brightness_level: {
                     enabled_by_default: false, entity_category: 'diagnostic', icon: 'mdi:brightness-5',
                 },
@@ -632,6 +697,9 @@ export default class HomeAssistant extends Extension {
                 smoke_density: {icon: 'mdi:google-circles-communities', state_class: 'measurement'},
                 soil_moisture: {icon: 'mdi:water-percent', state_class: 'measurement'},
                 temperature: {device_class: 'temperature', state_class: 'measurement'},
+                temperature_calibration: {entity_category: 'config', icon: 'mdi:wrench-clock'},
+                temperature_max: {entity_category: 'config', icon: 'mdi:thermometer-plus'},
+                temperature_min: {entity_category: 'config', icon: 'mdi:thermometer-minus'},
                 transition: {entity_category: 'config', icon: 'mdi:transition'},
                 voc: {device_class: 'volatile_organic_compounds', state_class: 'measurement'},
                 vibration_timeout: {entity_category: 'config', icon: 'mdi:timer'},
@@ -674,7 +742,7 @@ export default class HomeAssistant extends Extension {
                 discovery_payload: {
                     value_template: `{{ value_json.${firstExpose.property} }}`,
                     enabled_by_default: !allowsSet,
-                    ...(firstExpose.unit && {unit_of_measurement: firstExpose.unit}),
+                    unit_of_measurement: firstExpose.unit || '',
                     ...lookup[firstExpose.name],
                     ...extraAttrs,
                 },
@@ -697,10 +765,17 @@ export default class HomeAssistant extends Extension {
                         command_topic_prefix: endpoint,
                         command_topic_postfix: firstExpose.property,
                         ...(firstExpose.unit && {unit_of_measurement: firstExpose.unit}),
+                        ...(firstExpose.value_step && {step: firstExpose.value_step}),
                         ...lookup[firstExpose.name],
                     },
                 };
 
+                if (lookup[firstExpose.name]?.device_class === 'temperature') {
+                    discoveryEntry.discovery_payload.device_class == lookup[firstExpose.name]?.device_class;
+                } else {
+                    delete discoveryEntry.discovery_payload.device_class;
+                }
+
                 if (firstExpose.value_min != null) discoveryEntry.discovery_payload.min = firstExpose.value_min;
                 if (firstExpose.value_max != null) discoveryEntry.discovery_payload.max = firstExpose.value_max;
 
@@ -709,14 +784,20 @@ export default class HomeAssistant extends Extension {
         } else if (firstExpose.type === 'enum') {
             const lookup: {[s: string]: KeyValue} = {
                 action: {icon: 'mdi:gesture-double-tap'},
+                alarm_humidity: {icon: 'mdi:water-percent-alert'},
+                alarm_temperature: {icon: 'mdi:thermometer-alert'},
                 backlight_auto_dim: {entity_category: 'config', icon: 'mdi:brightness-auto'},
                 backlight_mode: {entity_category: 'config', icon: 'mdi:lightbulb'},
                 color_power_on_behavior: {entity_category: 'config', icon: 'mdi:palette'},
+                control_mode: {entity_category: 'config', icon: 'mdi:tune'},
                 device_mode: {entity_category: 'config', icon: 'mdi:tune'},
                 effect: {enabled_by_default: false, icon: 'mdi:palette'},
                 force: {enabled_by_default: false, icon: 'mdi:valve'},
                 keep_time: {entity_category: 'config', icon: 'mdi:av-timer'},
                 keypad_lockout: {entity_category: 'config', icon: 'mdi:lock'},
+                load_detection_mode: {entity_category: 'config', icon: 'mdi:tune'},
+                load_dimmable: {entity_category: 'diagnostic', icon: 'mdi:chart-bell-curve'},
+                load_type: {entity_category: 'diagnostic', icon: 'mdi:led-on'},
                 melody: {entity_category: 'config', icon: 'mdi:music-note'},
                 mode_phase_control: {entity_category: 'config', icon: 'mdi:tune'},
                 mode: {entity_category: 'config', icon: 'mdi:tune'},
@@ -724,53 +805,61 @@ export default class HomeAssistant extends Extension {
                 operation_mode: {entity_category: 'config', icon: 'mdi:tune'},
                 power_on_behavior: {entity_category: 'config', icon: 'mdi:power-settings'},
                 power_outage_memory: {entity_category: 'config', icon: 'mdi:power-settings'},
+                power_supply_mode: {entity_category: 'diagnostic', icon: 'mdi:power-settings'},
+                power_type: {entity_category: 'diagnostic', icon: 'mdi:lightning-bolt-circle'},
                 sensitivity: {entity_category: 'config', icon: 'mdi:tune'},
                 sensors_type: {entity_category: 'config', icon: 'mdi:tune'},
                 sound_volume: {entity_category: 'config', icon: 'mdi:volume-high'},
+                status: {icon: 'mdi:state-machine'},
                 switch_type: {entity_category: 'config', icon: 'mdi:tune'},
+                temperature_sensor_select: {entity_category: 'config', icon: 'mdi:home-thermometer'},
                 thermostat_unit: {entity_category: 'config', icon: 'mdi:thermometer'},
                 volume: {entity_category: 'config', icon: 'mdi: volume-high'},
                 week: {entity_category: 'config', icon: 'mdi:calendar-clock'},
             };
 
+            const valueTemplate = firstExpose.access & ACCESS_STATE ?
+                `{{ value_json.${firstExpose.property} }}` : undefined;
+
             if (firstExpose.access & ACCESS_STATE) {
                 discoveryEntries.push({
                     type: 'sensor',
                     object_id: firstExpose.property,
                     mockProperties: [{property: firstExpose.property, value: null}],
                     discovery_payload: {
-                        value_template: `{{ value_json.${firstExpose.property} }}`,
+                        value_template: valueTemplate,
                         enabled_by_default: !(firstExpose.access & ACCESS_SET),
                         ...lookup[firstExpose.name],
                     },
                 });
-
-                /**
-                 * If enum attribute has SET access then expose as SELECT entity too.
-                 * Note: currently both sensor and select are discovered, this is to avoid
-                 * breaking changes for sensors already existing in HA (legacy).
-                 */
-                if ((firstExpose.access & ACCESS_SET)) {
-                    discoveryEntries.push({
-                        type: 'select',
-                        object_id: firstExpose.property,
-                        mockProperties: [{property: firstExpose.property, value: null}],
-                        discovery_payload: {
-                            value_template: `{{ value_json.${firstExpose.property} }}`,
-                            state_topic: true,
-                            command_topic_prefix: endpoint,
-                            command_topic: true,
-                            command_topic_postfix: firstExpose.property,
-                            options: firstExpose.values.map((v) => v.toString()),
-                            ...lookup[firstExpose.name],
-                        },
-                    });
-                }
             }
-        } else if (firstExpose.type === 'text' || firstExpose.type === 'composite') {
+
+            /**
+             * If enum attribute has SET access then expose as SELECT entity too.
+             * Note: currently both sensor and select are discovered, this is to avoid
+             * breaking changes for sensors already existing in HA (legacy).
+             */
+            if ((firstExpose.access & ACCESS_SET)) {
+                discoveryEntries.push({
+                    type: 'select',
+                    object_id: firstExpose.property,
+                    mockProperties: [], // Already mocked above in case access STATE is supported
+                    discovery_payload: {
+                        value_template: valueTemplate,
+                        state_topic: !!(firstExpose.access & ACCESS_STATE),
+                        command_topic_prefix: endpoint,
+                        command_topic: true,
+                        command_topic_postfix: firstExpose.property,
+                        options: firstExpose.values.map((v) => v.toString()),
+                        ...lookup[firstExpose.name],
+                    },
+                });
+            }
+        } else if (firstExpose.type === 'text' || firstExpose.type === 'composite' || firstExpose.type === 'list') {
             if (firstExpose.access & ACCESS_STATE) {
                 const lookup: {[s: string]: KeyValue} = {
                     action: {icon: 'mdi:gesture-double-tap'},
+                    programming_mode: {icon: 'mdi:calendar-clock'},
                 };
 
                 const discoveryEntry: DiscoveryEntry = {
@@ -816,7 +905,11 @@ export default class HomeAssistant extends Extension {
         const entity = this.zigbee.resolveEntity(data.entity.name);
         if (entity.isDevice() && this.discovered[entity.ieeeAddr]) {
             for (const objectID of this.discovered[entity.ieeeAddr].objectIDs) {
-                const match = /light_(.*)/.exec(objectID);
+                const lightMatch = /light_(.*)/.exec(objectID);
+                const coverMatch = /cover_(.*)/.exec(objectID);
+
+                const match = lightMatch || coverMatch;
+
                 if (match) {
                     const endpoint = match[1];
                     const endpointRegExp = new RegExp(`(.*)_${endpoint}`);
@@ -863,7 +956,7 @@ export default class HomeAssistant extends Extension {
         }
     }
 
-    @bind onEntityRenamed(data: eventdata.EntityRenamed): void {
+    @bind async onEntityRenamed(data: eventdata.EntityRenamed): Promise<void> {
         logger.debug(`Refreshing Home Assistant discovery topic for '${data.entity.name}'`);
 
         // Clear before rename so Home Assistant uses new friendly_name
@@ -873,6 +966,10 @@ export default class HomeAssistant extends Extension {
                 const topic = this.getDiscoveryTopic(config, data.entity);
                 this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
             }
+
+            // Make sure Home Assistant deletes the old entity first otherwise another one (_2) is created
+            // https://github.com/Koenkk/zigbee2mqtt/issues/12610
+            await utils.sleep(2);
         }
 
         this.discover(data.entity, true);
@@ -893,8 +990,9 @@ export default class HomeAssistant extends Extension {
 
         let configs: DiscoveryEntry[] = [];
         if (isDevice) {
-            for (const expose of entity.exposes()) {
-                configs.push(...this.exposeToConfig([expose], 'device', entity.definition, entity.exposes()));
+            const exposes = entity.exposes(); // avoid calling it hundred of times/s
+            for (const expose of exposes) {
+                configs.push(...this.exposeToConfig([expose], 'device', exposes, entity.definition));
             }
 
             for (const mapping of legacyMapping) {
@@ -911,10 +1009,13 @@ export default class HomeAssistant extends Extension {
             }
         } else { // group
             const exposesByType: {[s: string]: zhc.DefinitionExpose[]} = {};
+            const allExposes: zhc.DefinitionExpose[] = [];
 
             entity.zh.members.map((e) => this.zigbee.resolveEntity(e.getDevice()) as Device)
                 .filter((d) => d.definition).forEach((device) => {
-                    for (const expose of device.exposes().filter((e) => groupSupportedTypes.includes(e.type))) {
+                    const exposes = device.exposes();
+                    allExposes.push(...exposes);
+                    for (const expose of exposes.filter((e) => groupSupportedTypes.includes(e.type))) {
                         let key = expose.type;
                         if (['switch', 'lock', 'cover'].includes(expose.type) && expose.endpoint) {
                             // A device can have multiple of these types which have to discovered seperately.
@@ -929,7 +1030,7 @@ export default class HomeAssistant extends Extension {
                 });
 
             configs = [].concat(...Object.values(exposesByType)
-                .map((exposes) => this.exposeToConfig(exposes, 'group')));
+                .map((exposes) => this.exposeToConfig(exposes, 'group', allExposes)));
         }
 
         if (isDevice && settings.get().advanced.last_seen !== 'disable') {
@@ -957,7 +1058,7 @@ export default class HomeAssistant extends Extension {
             const updateStateSensor: DiscoveryEntry = {
                 type: 'sensor',
                 object_id: 'update_state',
-                mockProperties: [{property: 'update', value: {state: null}}],
+                mockProperties: [], // update is mocked below with updateSensor
                 discovery_payload: {
                     icon: 'mdi:update',
                     value_template: `{{ value_json['update']['state'] }}`,
@@ -975,12 +1076,28 @@ export default class HomeAssistant extends Extension {
                     payload_on: true,
                     payload_off: false,
                     value_template: `{{ value_json['update']['state'] == "available" }}`,
-                    enabled_by_default: true,
+                    enabled_by_default: false,
                     device_class: 'update',
                     entity_category: 'diagnostic',
                 },
             };
             configs.push(updateAvailableSensor);
+            const updateSensor: DiscoveryEntry = {
+                type: 'update',
+                object_id: 'update',
+                mockProperties: [{property: 'update', value: {state: null}}],
+                discovery_payload: {
+                    entity_picture: 'https://github.com/Koenkk/zigbee2mqtt/raw/master/images/logo.png',
+                    latest_version_topic: true,
+                    state_topic: true,
+                    device_class: 'firmware',
+                    command_topic: `${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`,
+                    payload_install: `{"id": "${entity.ieeeAddr}"}`,
+                    value_template: `{{ value_json['update']['installed_version'] }}`,
+                    latest_version_template: `{{ value_json['update']['latest_version'] }}`,
+                },
+            };
+            configs.push(updateSensor);
         }
 
         if (isDevice && entity.options.hasOwnProperty('legacy') && !entity.options.legacy) {
@@ -1076,21 +1193,26 @@ export default class HomeAssistant extends Extension {
             // Availability payload
             payload.availability = [{topic: `${settings.get().mqtt.base_topic}/bridge/state`}];
 
-            const availabilityEnabled =
-                entity.isDevice() && utils.isAvailabilityEnabledForDevice(entity, settings.get());
             /* istanbul ignore next */
-            if (availabilityEnabled) {
+            if (utils.isAvailabilityEnabledForEntity(entity, settings.get())) {
                 payload.availability_mode = 'all';
                 payload.availability.push({topic: `${baseTopic}/availability`});
             }
 
+            if (entity.isDevice() && entity.options.disabled) {
+                // Mark disabled device always as unavailable
+                payload.availability.forEach((a: KeyValue) => a.value_template = '{{ "offline" }}');
+            } else if (!settings.get().advanced.legacy_availability_payload) {
+                payload.availability.forEach((a: KeyValue) => a.value_template = '{{ value_json.state }}');
+            }
+
             const commandTopicPrefix = payload.command_topic_prefix ? `${payload.command_topic_prefix}/` : '';
             delete payload.command_topic_prefix;
             const commandTopicPostfix = payload.command_topic_postfix ? `/${payload.command_topic_postfix}` : '';
             delete payload.command_topic_postfix;
             const commandTopic = `${baseTopic}/${commandTopicPrefix}set${commandTopicPostfix}`;
 
-            if (payload.command_topic) {
+            if (payload.command_topic && typeof payload.command_topic !== 'string') {
                 payload.command_topic = commandTopic;
             }
 
@@ -1110,22 +1232,6 @@ export default class HomeAssistant extends Extension {
                 payload.mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/system_mode`;
             }
 
-            if (payload.hold_command_topic) {
-                payload.hold_command_topic = `${baseTopic}/${commandTopicPrefix}set/preset`;
-            }
-
-            if (payload.hold_state_topic) {
-                payload.hold_state_topic = stateTopic;
-            }
-
-            if (payload.away_mode_state_topic) {
-                payload.away_mode_state_topic = stateTopic;
-            }
-
-            if (payload.away_mode_command_topic) {
-                payload.away_mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/away_mode`;
-            }
-
             if (payload.current_temperature_topic) {
                 payload.current_temperature_topic = stateTopic;
             }
@@ -1161,10 +1267,22 @@ export default class HomeAssistant extends Extension {
                 payload.fan_mode_state_topic = stateTopic;
             }
 
+            if (payload.latest_version_topic) {
+                payload.latest_version_topic = stateTopic;
+            }
+
             if (payload.fan_mode_command_topic) {
                 payload.fan_mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/fan_mode`;
             }
 
+            if (payload.swing_mode_state_topic) {
+                payload.swing_mode_state_topic = stateTopic;
+            }
+
+            if (payload.swing_mode_command_topic) {
+                payload.swing_mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/swing_mode`;
+            }
+
             if (payload.percentage_state_topic) {
                 payload.percentage_state_topic = stateTopic;
             }
@@ -1178,7 +1296,8 @@ export default class HomeAssistant extends Extension {
             }
 
             if (payload.preset_mode_command_topic) {
-                payload.preset_mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/fan_mode`;
+                payload.preset_mode_command_topic = `${baseTopic}/${commandTopicPrefix}set/` +
+                    payload.preset_mode_command_topic;
             }
 
             if (payload.action_topic) {
@@ -1280,10 +1399,9 @@ export default class HomeAssistant extends Extension {
             data.message.toLowerCase() === 'online') {
             const timer = setTimeout(async () => {
                 // Publish all device states.
-                this.mqtt.publish('bridge/state', 'online', {retain: true, qos: 0});
-                for (const device of this.zigbee.devices(false)) {
-                    if (this.state.exists(device)) {
-                        this.publishEntityState(device, this.state.get(device));
+                for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+                    if (this.state.exists(entity)) {
+                        this.publishEntityState(entity, this.state.get(entity), 'publishCached');
                     }
                 }
 
@@ -1344,6 +1462,10 @@ export default class HomeAssistant extends Extension {
                 message.color.s = message.color.saturation;
             }
         }
+
+        if (entity.isDevice() && entity.definition?.ota && message.update?.latest_version == null) {
+            message.update = {...message.update, installed_version: -1, latest_version: -1};
+        }
     }
 
     private getEncodedBaseTopic(): string {

lib/extension/networkMap.ts

@@ -214,7 +214,7 @@ export default class NetworkMap extends Extension {
 
     async networkScan(includeRoutes: boolean): Promise<Topology> {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
-        const devices = this.zigbee.devices().filter((d) => d.zh.type !== 'GreenPower');
+        const devices = this.zigbee.devices().filter((d) => d.zh.type !== 'GreenPower' && !d.options.disabled);
         const lqis: Map<Device, zh.LQI> = new Map();
         const routingTables: Map<Device, zh.RoutingTable> = new Map();
         const failed: Map<Device, string[]> = new Map();
@@ -244,6 +244,7 @@ export default class NetworkMap extends Extension {
             } catch (error) {
                 failed.get(device).push('lqi');
                 logger.error(`Failed to execute LQI for '${device.name}'`);
+                logger.debug(error.stack);
             }
 
             if (includeRoutes) {

lib/extension/onEvent.ts

@@ -40,10 +40,11 @@ export default class OnEvent extends Extension {
     }
 
     private async callOnEvent(device: Device, type: string, data: KeyValue): Promise<void> {
-        zhc.onEvent(type, data, device.zh);
+        const state = this.state.get(device);
+        zhc.onEvent(type, data, device.zh, device.options, state);
 
         if (device.definition?.onEvent) {
-            await device.definition.onEvent(type, data, device.zh, device.options);
+            await device.definition.onEvent(type, data, device.zh, device.options, state);
         }
     }
 }

lib/extension/otaUpdate.ts

@@ -24,8 +24,13 @@ function isValidUrl(url: string): boolean {
 
 type UpdateState = 'updating' | 'idle' | 'available';
 interface UpdatePayload {
+    update_available?: boolean
     // eslint-disable-next-line camelcase
-    update: {progress?: number, remaining?: number, state: UpdateState}, update_available?: boolean}
+    update: {
+        progress?: number, remaining?: number, state: UpdateState,
+        installed_version: number | null, latest_version: number | null
+    }
+}
 
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
 const topicRegex =
@@ -65,38 +70,39 @@ export default class OTAUpdate extends Extension {
     }
 
     private removeProgressAndRemainingFromState(device: Device): void {
-        delete this.state.get(device)?.update?.progress;
-        delete this.state.get(device)?.update?.remaining;
+        delete this.state.get(device).update?.progress;
+        delete this.state.get(device).update?.remaining;
     }
 
     @bind private async onZigbeeEvent(data: eventdata.DeviceMessage): Promise<void> {
-        if (settings.get().ota.disable_automatic_update_check) return;
-        if (data.type !== 'commandQueryNextImageRequest' || !data.device.definition) return;
+        if (data.type !== 'commandQueryNextImageRequest' || !data.device.definition ||
+            this.inProgress.has(data.device.ieeeAddr)) return;
         logger.debug(`Device '${data.device.name}' requested OTA`);
 
+        const automaticOTACheckDisabled = settings.get().ota.disable_automatic_update_check;
         let supportsOTA = data.device.definition.hasOwnProperty('ota');
-        if (supportsOTA) {
+        if (supportsOTA && !automaticOTACheckDisabled) {
             // When a device does a next image request, it will usually do it a few times after each other
             // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
             // each time, so this interval can be set by the user. The default is 1,440 minutes (one day).
             const updateCheckInterval = settings.get().ota.update_check_interval * 1000 * 60;
             const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
                 (Date.now() - this.lastChecked[data.device.ieeeAddr]) > updateCheckInterval : true;
-            if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
+            if (!check) return;
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
-            let available = false;
+            let availableResult: zhc.OtaUpdateAvailableResult = null;
             try {
-                available = await data.device.definition.ota.isUpdateAvailable(data.device.zh, logger, data.data);
+                availableResult = await data.device.definition.ota.isUpdateAvailable(data.device.zh, logger, data.data);
             } catch (e) {
                 supportsOTA = false;
                 logger.debug(`Failed to check if update available for '${data.device.name}' (${e.message})`);
             }
 
-            const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+            const payload = this.getEntityPublishPayload(data.device, availableResult ?? 'idle');
             this.publishEntityState(data.device, payload);
 
-            if (available) {
+            if (availableResult?.available) {
                 const message = `Update available for '${data.device.name}'`;
                 logger.info(message);
 
@@ -111,13 +117,11 @@ export default class OTAUpdate extends Extension {
             }
         }
 
-        // Respond to the OTA request:
-        // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
-        // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
+        // Respond to the OTA request: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
         const endpoint = data.device.zh.endpoints.find((e) => e.supportsOutputCluster('genOta'));
         if (endpoint) {
             // Some devices send OTA requests without defining OTA cluster as input cluster.
-            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: 0x98});
         }
     }
 
@@ -132,14 +136,22 @@ export default class OTAUpdate extends Extension {
         }
     }
 
-    private getEntityPublishPayload(state: UpdateState, progress: number=null, remaining: number=null): UpdatePayload {
-        const payload: UpdatePayload = {update: {state}};
+    private getEntityPublishPayload(device: Device, state: zhc.OtaUpdateAvailableResult | UpdateState,
+        progress: number=null, remaining: number=null): UpdatePayload {
+        const deviceUpdateState = this.state.get(device).update;
+        const payload: UpdatePayload = {update: {
+            state: typeof state === 'string' ? state : (state.available ? 'available' : 'idle'),
+            installed_version: typeof state === 'string' ?
+                deviceUpdateState?.installed_version : state.currentFileVersion,
+            latest_version: typeof state === 'string' ?
+                deviceUpdateState?.latest_version : state.otaFileVersion,
+        }};
         if (progress !== null) payload.update.progress = progress;
         if (remaining !== null) payload.update.remaining = Math.round(remaining);
 
         /* istanbul ignore else */
         if (this.legacyApi) {
-            payload.update_available = state === 'available';
+            payload.update_available = typeof state === 'string' ? state === 'available' : state.available;
         }
 
         return payload;
@@ -190,23 +202,24 @@ export default class OTAUpdate extends Extension {
                 }
 
                 try {
-                    const available = await device.definition.ota.isUpdateAvailable(device.zh, logger);
-                    const msg = `${available ? 'Update' : 'No update'} available for '${device.name}'`;
+                    const availableResult = await device.definition.ota.isUpdateAvailable(device.zh, logger);
+                    const msg = `${availableResult.available ? 'Update' : 'No update'} available for '${device.name}'`;
                     logger.info(msg);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: available ? 'available' : 'not_available', device: device.name};
+                        const meta = {
+                            status: availableResult.available ? 'available' : 'not_available', device: device.name};
                         this.mqtt.publish(
                             'bridge/log',
                             stringify({type: `ota_update`, message: msg, meta}),
                         );
                     }
 
-                    const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+                    const payload = this.getEntityPublishPayload(device, availableResult);
                     this.publishEntityState(device, payload);
                     this.lastChecked[device.ieeeAddr] = Date.now();
-                    responseData.updateAvailable = available;
+                    responseData.updateAvailable = availableResult.available;
                 } catch (e) {
                     error = `Failed to check if update available for '${device.name}' (${e.message})`;
                     errorStack = e.stack;
@@ -242,7 +255,7 @@ export default class OTAUpdate extends Extension {
 
                         logger.info(msg);
 
-                        const payload = this.getEntityPublishPayload('updating', progress, remaining);
+                        const payload = this.getEntityPublishPayload(device, 'updating', progress, remaining);
                         this.publishEntityState(device, payload);
 
                         /* istanbul ignore else */
@@ -253,11 +266,12 @@ export default class OTAUpdate extends Extension {
                     };
 
                     const from_ = await this.readSoftwareBuildIDAndDateCode(device, 'immediate');
-                    await device.definition.ota.updateToLatest(device.zh, logger, onProgress);
+                    const fileVersion = await device.definition.ota.updateToLatest(device.zh, logger, onProgress);
                     logger.info(`Finished update of '${device.name}'`);
                     this.eventBus.emitReconfigure({device});
                     this.removeProgressAndRemainingFromState(device);
-                    const payload = this.getEntityPublishPayload('idle');
+                    const payload = this.getEntityPublishPayload(device,
+                        {available: false, currentFileVersion: fileVersion, otaFileVersion: fileVersion});
                     this.publishEntityState(device, payload);
                     const to = await this.readSoftwareBuildIDAndDateCode(device, 'active');
                     const [fromS, toS] = [stringify(from_), stringify(to)];
@@ -277,7 +291,7 @@ export default class OTAUpdate extends Extension {
                     errorStack = e.stack;
 
                     this.removeProgressAndRemainingFromState(device);
-                    const payload = this.getEntityPublishPayload('available');
+                    const payload = this.getEntityPublishPayload(device, 'available');
                     this.publishEntityState(device, payload);
 
                     /* istanbul ignore else */

lib/extension/publish.ts

@@ -135,7 +135,7 @@ export default class Publish extends Extension {
         }
         const device = re instanceof Device ? re.zh : null;
         const entitySettings = re.options;
-        const entityState = this.state.get(re) || {};
+        const entityState = this.state.get(re);
         const membersState = re instanceof Group ?
             Object.fromEntries(re.zh.members.map((e) => [e.getDevice().ieeeAddr,
                 this.state.get(this.zigbee.resolveEntity(e.getDevice().ieeeAddr))])) : null;
@@ -256,7 +256,8 @@ export default class Publish extends Extension {
                         }
 
                         // filter out attribute listed in filtered_optimistic
-                        entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
+                        utils.filterProperties(entitySettings.filtered_optimistic, msg);
+
                         addToToPublish(re, msg);
                     }
 

lib/extension/receive.ts

@@ -145,9 +145,9 @@ export default class Receive extends Extension {
                 if (converted) {
                     payload = {...payload, ...converted};
                 }
-            } catch (error) {
-                // istanbul ignore next
+            } catch (error) /* istanbul ignore next */ {
                 logger.error(`Exception while calling fromZigbee converter: ${error.message}}`);
+                logger.debug(error.stack);
             }
         }
 

lib/model/device.ts

@@ -5,6 +5,7 @@ import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
 export default class Device {
     public zh: zh.Device;
     private _definition: zhc.Definition;
+    private _definitionModelID: string;
 
     get ieeeAddr(): string {return this.zh.ieeeAddr;}
     get ID(): string {return this.zh.ieeeAddr;}
@@ -13,8 +14,11 @@ export default class Device {
         return this.zh.type === 'Coordinator' ? 'Coordinator' : this.options?.friendly_name || this.ieeeAddr;
     }
     get definition(): zhc.Definition {
-        if (!this._definition && !this.zh.interviewing) {
+        // Some devices can change modelID, reconsider the definition in that case.
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/3016
+        if (!this.zh.interviewing && (!this._definition || this._definitionModelID !== this.zh.modelID)) {
             this._definition = zigbeeHerdsmanConverters.findByDevice(this.zh);
+            this._definitionModelID = this.zh.modelID;
         }
         return this._definition;
     }

lib/model/group.ts

@@ -4,13 +4,23 @@ import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
 
 export default class Group {
     public zh: zh.Group;
+    private resolveDevice: (ieeeAddr: string) => Device;
 
     get ID(): number {return this.zh.groupID;}
-    get options(): GroupOptions {return settings.getGroup(this.ID);}
+    get options(): GroupOptions {return {...settings.getGroup(this.ID)};}
     get name(): string {return this.options?.friendly_name || this.ID.toString();}
 
-    constructor(group: zh.Group) {
+    constructor(group: zh.Group, resolveDevice: (ieeeAddr: string) => Device) {
         this.zh = group;
+        this.resolveDevice = resolveDevice;
+    }
+
+    hasMember(device: Device): boolean {
+        return !!device.zh.endpoints.find((e) => this.zh.members.includes(e));
+    }
+
+    membersDevices(): Device[] {
+        return this.zh.members.map((e) => this.resolveDevice(e.getDevice().ieeeAddr)).filter((d) => d);
     }
 
     membersDefinitions(): zhc.Definition[] {

lib/mqtt.ts

@@ -10,6 +10,10 @@ export default class MQTT {
     private connectionTimer: NodeJS.Timeout;
     private client: mqtt.MqttClient;
     private eventBus: EventBus;
+    private initialConnect = true;
+    private republishRetainedTimer: NodeJS.Timer;
+    private retainedMessages: {[s: string]: {payload: string, options: MQTTOptions,
+        skipLog: boolean, skipReceive: boolean, topic: string, base: string}} = {};
 
     constructor(eventBus: EventBus) {
         this.eventBus = eventBus;
@@ -22,7 +26,7 @@ export default class MQTT {
         const options: mqtt.IClientOptions = {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
-                payload: 'offline',
+                payload: utils.availabilityPayload('offline', settings.get()),
                 retain: settings.get().mqtt.force_disable_retain ? false : true,
                 qos: 1,
             },
@@ -71,6 +75,7 @@ export default class MQTT {
             this.client = mqtt.connect(mqttSettings.server, options);
             // @ts-ignore https://github.com/Koenkk/zigbee2mqtt/issues/9822
             this.client.stream.setMaxListeners(0);
+            this.eventBus.onPublishAvailability(this, () => this.publishStateOnline());
 
             const onConnect = this.onConnect;
             this.client.on('connect', async () => {
@@ -78,7 +83,10 @@ export default class MQTT {
                 resolve();
             });
 
-            this.client.on('error', (err) => reject(err));
+            this.client.on('error', (err) => {
+                logger.error(`MQTT error: ${err.message}`);
+                reject(err);
+            });
             this.client.on('message', this.onMessage);
         });
     }
@@ -93,13 +101,30 @@ export default class MQTT {
         }, utils.seconds(10));
 
         logger.info('Connected to MQTT server');
+        await this.publishStateOnline();
+
+        if (!this.initialConnect) {
+            this.republishRetainedTimer = setTimeout(() => {
+                // Republish retained messages in case MQTT broker does not persist them.
+                // https://github.com/Koenkk/zigbee2mqtt/issues/9629
+                Object.values(this.retainedMessages).forEach((e) =>
+                    this.publish(e.topic, e.payload, e.options, e.base, e.skipLog, e.skipReceive));
+            }, 2000);
+        }
+
+        this.initialConnect = false;
         this.subscribe(`${settings.get().mqtt.base_topic}/#`);
-        await this.publish('bridge/state', 'online', {retain: true, qos: 0});
+    }
+
+    async publishStateOnline(): Promise<void> {
+        await this.publish('bridge/state', utils.availabilityPayload('online', settings.get()), {retain: true, qos: 0});
     }
 
     async disconnect(): Promise<void> {
         clearTimeout(this.connectionTimer);
-        await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
+        await this.publish('bridge/state', utils.availabilityPayload('offline', settings.get()),
+            {retain: true, qos: 0});
+        this.eventBus.removeListeners(this);
         logger.info('Disconnecting from MQTT server');
         this.client.end();
     }
@@ -114,6 +139,11 @@ export default class MQTT {
             logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
             this.eventBus.emitMQTTMessage({topic, message: message + ''});
         }
+
+        if (this.republishRetainedTimer && topic == `${settings.get().mqtt.base_topic}/bridge/state`) {
+            clearTimeout(this.republishRetainedTimer);
+            this.republishRetainedTimer = null;
+        }
     }
 
     isConnected(): boolean {
@@ -130,6 +160,15 @@ export default class MQTT {
             this.publishedTopics.add(topic);
         }
 
+        if (options.retain) {
+            if (payload) {
+                this.retainedMessages[topic] =
+                    {payload, options, skipReceive, skipLog, topic: topic.substring(base.length + 1), base};
+            } else {
+                delete this.retainedMessages[topic];
+            }
+        }
+
         this.eventBus.emitMQTTMessagePublished({topic, payload, options: {...defaultOptions, ...options}});
 
         if (!this.isConnected()) {

lib/state.ts

@@ -1,25 +1,26 @@
 import logger from './util/logger';
 import data from './util/data';
 import * as settings from './util/settings';
+import utils from './util/utils';
 import fs from 'fs';
 import objectAssignDeep from 'object-assign-deep';
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
-    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
-    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];
 
 class State {
     private state: {[s: string | number]: KeyValue} = {};
     private file = data.joinPath('state.json');
     private timer: NodeJS.Timer = null;
-    private eventBus: EventBus;
 
-    constructor(eventBus: EventBus) {
+    constructor(private readonly eventBus: EventBus, private readonly zigbee: Zigbee) {
         this.eventBus = eventBus;
+        this.zigbee = zigbee;
     }
 
     start(): void {
@@ -27,11 +28,14 @@ class State {
 
         // Save the state on every interval
         this.timer = setInterval(() => this.save(), saveInterval);
-        this.eventBus.onDeviceLeave(this, (data) => delete this.state[data.ieeeAddr]);
     }
 
     stop(): void {
-        this.eventBus.removeListeners(this);
+        // Remove any invalid states (ie when the device has left the network) when the system is stopped
+        Object.keys(this.state)
+            .filter((k) => typeof k === 'string' && !this.zigbee.resolveEntity(k)) // string key = ieeeAddr
+            .forEach((k) => delete this.state[k]);
+
         clearTimeout(this.timer);
         this.save();
     }
@@ -68,23 +72,20 @@ class State {
     }
 
     get(entity: Group | Device): KeyValue {
-        return this.state[entity.ID];
+        return this.state[entity.ID] || {};
     }
 
     set(entity: Group | Device, update: KeyValue, reason: string=null): KeyValue {
         const fromState = this.state[entity.ID] || {};
         const toState = objectAssignDeep({}, fromState, update);
-        const result = {...toState};
+        const newCache = {...toState};
+        const entityDontCacheProperties = entity.options.filtered_cache || [];
 
-        for (const property of Object.keys(toState)) {
-            if (dontCacheProperties.find((p) => property.match(p))) {
-                delete toState[property];
-            }
-        }
+        utils.filterProperties(dontCacheProperties.concat(entityDontCacheProperties), newCache);
 
-        this.state[entity.ID] = toState;
+        this.state[entity.ID] = newCache;
         this.eventBus.emitStateChange({entity, from: fromState, to: toState, reason, update});
-        return result;
+        return toState;
     }
 
     remove(ID: string | number): void {

lib/types/types.d.ts

@@ -42,7 +42,7 @@ declare global {
     // Types
     interface MQTTResponse {data: KeyValue, status: 'error' | 'ok', error?: string, transaction?: string}
     interface MQTTOptions {qos?: mqtt.QoS, retain?: boolean, properties?: {messageExpiryInterval: number}}
-    type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged';
+    type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged' | 'publishCached';
     type PublishEntityState = (entity: Device | Group, payload: KeyValue,
         stateChangeReason?: StateChangeReason) => Promise<void>;
     type RecursivePartial<T> = {[P in keyof T]?: RecursivePartial<T[P]>;};
@@ -100,10 +100,12 @@ declare global {
 
         interface DefinitionExpose {
             type: string, name?: string, features?: DefinitionExposeFeature[],
-            endpoint?: string, values?: string[], value_off?: string, value_on?: string,
+            endpoint?: string, values?: string[], value_off?: string, value_on?: string, value_step?: number,
             access: number, property: string, unit?: string,
             value_min?: number, value_max?: number}
 
+        interface OtaUpdateAvailableResult {available: boolean, currentFileVersion: number, otaFileVersion: number}
+
         interface Definition {
             model: string,
             zigbeeModel: string[],
@@ -117,11 +119,13 @@ declare global {
             exposes: DefinitionExpose[] | ((device: zh.Device, options: KeyValue) => DefinitionExpose[])
             configure?: (device: zh.Device, coordinatorEndpoint: zh.Endpoint, logger: Logger,
                 options?: DeviceOptions) => Promise<void>;
-            onEvent?: (type: string, data: KeyValue, device: zh.Device, settings: KeyValue) => Promise<void>;
+            onEvent?: (type: string, data: KeyValue, device: zh.Device,
+                settings: KeyValue, state: KeyValue) => Promise<void>;
             ota?: {
-                isUpdateAvailable: (device: zh.Device, logger: Logger, data?: KeyValue) => Promise<boolean>;
+                isUpdateAvailable: (device: zh.Device, logger: Logger, data?: KeyValue)
+                    => Promise<OtaUpdateAvailableResult>;
                 updateToLatest: (device: zh.Device, logger: Logger,
-                    onProgress: (progress: number, remaining: number) => void) => Promise<void>;
+                    onProgress: (progress: number, remaining: number) => void) => Promise<number>;
             }
         }
 
@@ -232,12 +236,15 @@ declare global {
             host?: string,
             port?: number,
             url?: string,
+            ssl_cert?: string,
+            ssl_key?: string,
         },
         devices?: {[s: string]: DeviceOptions},
         groups?: {[s: string]: GroupOptions},
         device_options: KeyValue,
         advanced: {
             legacy_api: boolean,
+            legacy_availability_payload: boolean,
             log_rotation: boolean,
             log_symlink_current: boolean,
             log_output: ('console' | 'file' | 'syslog')[],
@@ -272,18 +279,21 @@ declare global {
 
     interface DeviceOptions {
         ID?: string,
+        disabled?: boolean,
         retention?: number,
         availability?: boolean | {timeout: number},
         optimistic?: boolean,
         retrieve_state?: boolean,
         debounce?: number,
         debounce_ignore?: string[],
+        filtered_attributes?: string[],
+        filtered_cache?: string[],
         filtered_optimistic?: string[],
         icon?: string,
         homeassistant?: KeyValue,
         legacy?: boolean,
-        filtered_attributes?: string[],
         friendly_name: string,
+        description?: string,
         qos?: 0 | 1 | 2,
     }
 
@@ -291,11 +301,14 @@ declare global {
         devices?: string[],
         ID?: number,
         optimistic?: boolean,
+        off_state?: 'all_members_off' | 'last_member_state'
+        filtered_attributes?: string[],
+        filtered_cache?: string[],
         filtered_optimistic?: string[],
         retrieve_state?: boolean,
         homeassistant?: KeyValue,
-        filtered_attributes?: string[],
         friendly_name: string,
+        description?: string,
         qos?: 0 | 1 | 2,
     }
 }

lib/types/zigbee-herdsman-converters.d.ts

@@ -4,7 +4,8 @@ declare module 'zigbee-herdsman-converters' {
     export const toZigbeeConverters: {[s: string]: zhc.ToZigbeeConverter};
     export const definitions: zhc.Definition[];
     export function addDeviceDefinition(definition: zhc.Definition): Promise<void>;
-    export function onEvent(type: string, data: KeyValue, device: zh.Device): Promise<void>;
+    export function onEvent(type: string, data: KeyValue, device: zh.Device, settings: KeyValue,
+        state: KeyValue): Promise<void>;
 }
 
 declare module 'zigbee-herdsman-converters/lib/ota/tradfri' {

lib/util/data.ts

@@ -14,7 +14,7 @@ function load(): void {
 load();
 
 function joinPath(file: string): string {
-    return path.join(dataPath, file);
+    return path.resolve(dataPath, file);
 }
 
 function getPath(): string {

lib/util/logger.ts

@@ -9,40 +9,12 @@ import assert from 'assert';
 
 const colorizer = winston.format.colorize();
 
-// What transports to enable
-const output = settings.get().advanced.log_output;
-
-// Directory to log to
-const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
-const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
-const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
-
-// Make sure that log directoy exsists when not logging to stdout only
-if (output.includes('file')) {
-    fx.mkdirSync(directory);
-
-    if (settings.get().advanced.log_symlink_current) {
-        const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
-        const actual = './' + timestamp;
-        if (fs.existsSync(current)) {
-            fs.unlinkSync(current);
-        }
-        fs.symlinkSync(actual, current);
-    }
-}
-
 type Z2MLogLevel = 'warn' | 'debug' | 'info' | 'error';
 type WinstonLogLevel = 'warning' | 'debug' | 'info' | 'error';
 
 const z2mToWinstonLevel = (level: Z2MLogLevel): WinstonLogLevel => level === 'warn' ? 'warning' : level;
 const winstonToZ2mLevel = (level: WinstonLogLevel): Z2MLogLevel => level === 'warning' ? 'warn' : level;
 
-// Determine the log level.
-const z2mLevel = settings.get().advanced.log_level;
-const validLevels = ['info', 'error', 'warn', 'debug'];
-assert(validLevels.includes(z2mLevel), `'${z2mLevel}' is not valid log_level, use one of '${validLevels.join(', ')}'`);
-const level = z2mToWinstonLevel(z2mLevel);
-
 const levelWithCompensatedLength: {[s: string]: string} = {
     'info': 'info ',
     'error': 'error',
@@ -50,71 +22,113 @@ const levelWithCompensatedLength: {[s: string]: string} = {
     'debug': 'debug',
 };
 
-const timestampFormat = (): string => moment().format(settings.get().advanced.timestamp_format);
+let logger: winston.Logger;
+let fileTransport : winston.transport;
+let output: string[];
+let directory: string;
+let logFilename: string;
+let transportsToUse: winston.transport[];
 
-// Setup default console logger
-const transportsToUse: winston.transport[] = [
-    new winston.transports.Console({
+function init(): void {
+    // What transports to enable
+    output = settings.get().advanced.log_output;
+
+    // Directory to log to
+    const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
+    directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+    logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
+
+    // Make sure that log directoy exsists when not logging to stdout only
+    if (output.includes('file')) {
+        fx.mkdirSync(directory);
+
+        if (settings.get().advanced.log_symlink_current) {
+            const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
+            const actual = './' + timestamp;
+            /* istanbul ignore next */
+            if (fs.existsSync(current)) {
+                fs.unlinkSync(current);
+            }
+            fs.symlinkSync(actual, current);
+        }
+    }
+
+    // Determine the log level.
+    const z2mLevel = settings.get().advanced.log_level;
+    const validLevels = ['info', 'error', 'warn', 'debug'];
+    assert(validLevels.includes(z2mLevel),
+        `'${z2mLevel}' is not valid log_level, use one of '${validLevels.join(', ')}'`);
+    const level = z2mToWinstonLevel(z2mLevel);
+
+    const timestampFormat = (): string => moment().format(settings.get().advanced.timestamp_format);
+
+    // Setup default console logger
+    transportsToUse = [
+        new winston.transports.Console({
+            level,
+            silent: !output.includes('console'),
+            format: winston.format.combine(
+                winston.format.timestamp({format: timestampFormat}),
+                winston.format.printf(/* istanbul ignore next */(info) => {
+                    const {timestamp, level, message} = info;
+                    const l = winstonToZ2mLevel(level as WinstonLogLevel);
+
+                    const plainPrefix = `Zigbee2MQTT:${levelWithCompensatedLength[l]}`;
+                    let prefix = plainPrefix;
+                    if (process.stdout.isTTY) {
+                        prefix = colorizer.colorize(l, plainPrefix);
+                    }
+                    return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+                }),
+            ),
+        }),
+    ];
+
+    // Add file logger when enabled
+    // NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+    const transportFileOptions: KeyValue = {
+        filename: path.join(directory, logFilename),
+        json: false,
         level,
-        silent: !output.includes('console'),
         format: winston.format.combine(
             winston.format.timestamp({format: timestampFormat}),
             winston.format.printf(/* istanbul ignore next */(info) => {
                 const {timestamp, level, message} = info;
                 const l = winstonToZ2mLevel(level as WinstonLogLevel);
-                const prefix = colorizer.colorize(l, `Zigbee2MQTT:${levelWithCompensatedLength[l]}`);
-                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+                return `${levelWithCompensatedLength[l]} ${timestamp.split('.')[0]}: ${message}`;
             }),
         ),
-    }),
-];
-
-// Add file logger when enabled
-// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
-const transportFileOptions: KeyValue = {
-    filename: path.join(directory, logFilename),
-    json: false,
-    level,
-    format: winston.format.combine(
-        winston.format.timestamp({format: timestampFormat}),
-        winston.format.printf(/* istanbul ignore next */(info) => {
-            const {timestamp, level, message} = info;
-            const l = winstonToZ2mLevel(level as WinstonLogLevel);
-            return `${levelWithCompensatedLength[l]} ${timestamp.split('.')[0]}: ${message}`;
-        }),
-    ),
-};
-
-if (settings.get().advanced.log_rotation) {
-    transportFileOptions.tailable = true;
-    transportFileOptions.maxFiles = 3; // Keep last 3 files
-    transportFileOptions.maxsize = 10000000; // 10MB
-}
-
-let fileTransport : winston.transport;
-if (output.includes('file')) {
-    fileTransport = new winston.transports.File(transportFileOptions);
-    transportsToUse.push(fileTransport);
-}
-
-/* istanbul ignore next */
-if (output.includes('syslog')) {
-    // eslint-disable-next-line
-    require('winston-syslog').Syslog;
-    const options: KeyValue = {
-        app_name: 'Zigbee2MQTT',
-        format: winston.format.printf(/* istanbul ignore next */(info) => {
-            return `${info.message}`;
-        }),
-        ...settings.get().advanced.log_syslog,
     };
-    if (options.hasOwnProperty('type')) options.type = options.type.toString();
-    // @ts-ignore
-    transportsToUse.push(new winston.transports.Syslog(options));
-}
 
-// Create logger
-const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
+    if (settings.get().advanced.log_rotation) {
+        transportFileOptions.tailable = true;
+        transportFileOptions.maxFiles = 3; // Keep last 3 files
+        transportFileOptions.maxsize = 10000000; // 10MB
+    }
+
+    if (output.includes('file')) {
+        fileTransport = new winston.transports.File(transportFileOptions);
+        transportsToUse.push(fileTransport);
+    }
+
+    /* istanbul ignore next */
+    if (output.includes('syslog')) {
+        // eslint-disable-next-line
+        require('winston-syslog').Syslog;
+        const options: KeyValue = {
+            app_name: 'Zigbee2MQTT',
+            format: winston.format.printf(/* istanbul ignore next */(info) => {
+                return `${info.message}`;
+            }),
+            ...settings.get().advanced.log_syslog,
+        };
+        if (options.hasOwnProperty('type')) options.type = options.type.toString();
+        // @ts-ignore
+        transportsToUse.push(new winston.transports.Syslog(options));
+    }
+
+    logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
+}
 
 // Cleanup any old log directory.
 function cleanup(): void {
@@ -206,5 +220,6 @@ async function end(): Promise<void> {
 }
 
 export default {
-    logOutput, warn, warning, error, info, debug, setLevel, getLevel, cleanup, addTransport, end, winston: logger,
+    init, logOutput, warn, warning, error, info, debug, setLevel, getLevel, cleanup, addTransport, end,
+    winston: (): winston.Logger => logger,
 };

lib/util/settings.schema.json

@@ -388,6 +388,18 @@
               "title": "URL",
               "description": "URL on which the frontend can be reached, currently only used for the Home Assistant device configuration page",
               "requiresRestart": true
+            },
+            "ssl_cert" :  {
+              "type": ["string", "null"],
+              "title": "Certificate file path",
+              "description": "SSL Certificate file path for exposing HTTPS. The sibling property 'ssl_key' must be set for HTTPS to be activated.",
+              "requiresRestart": true
+            },
+            "ssl_key" :  {
+              "type": ["string", "null"],
+              "title": "key file path",
+              "description": "SSL key file path for exposing HTTPS. The sibling property 'ssl_cert' must be set for HTTPS to be activated.",
+              "requiresRestart": true
             }
           }
         }
@@ -432,6 +444,13 @@
           "description": "Disables the legacy api (false = disable)",
           "default": true
         },
+        "legacy_availability_payload": {
+          "type": "boolean",
+          "title": "Legacy availability payload",
+          "requiresRestart": true,
+          "description": "Payload to be used for device availabilty and bridge/state topics. true = text, false = JSON",
+          "default": true
+        },
         "log_rotation": {
           "type": "boolean",
           "title": "Log rotation",
@@ -775,6 +794,12 @@
           "title": "Retain",
           "description": "Retain MQTT messages of this device"
         },
+        "disabled": {
+          "type": "boolean",
+          "title": "Disabled",
+          "description": "Disables the device (excludes device from network scans, availability and group state updates)",
+          "requiresRestart": true
+        },
         "retention": {
           "type": "number",
           "title": "Retention",
@@ -805,28 +830,48 @@
           "description": "Publish optimistic state after set",
           "default": true
         },
-        "filtered_optimistic": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "examples": ["color_mode", "color_temp", "color"],
-          "title": "Filtered optimistic attributes",
-          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
-        },
         "filtered_attributes": {
           "type": "array",
           "items": {
             "type": "string"
           },
-          "examples": ["temperature", "battery", "action"],
+          "examples": ["^temperature$", "^battery$", "^action$"],
           "title": "Filtered publish attributes",
-          "description": "Filter attributes from publish payload."
+          "description": "Filter attributes with regex from published payload."
+        },
+        "filtered_cache": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["^input_actions$"],
+          "title": "Filtered attributes from cache",
+          "description": "Filter attributes with regex from being added to the cache, this prevents the attribute from being in the published payload when the value didn't change."
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["^color_(mode|temp)$", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes with regex from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
         },
         "icon": {
           "type": "string",
           "title": "Icon",
-          "description": "The user-defined device icon for the frontend. It can be a link to an image (not a path to a file) or base64 encoded data URL like: image/svg+xml;base64,PHN2ZyB3aW....R0aD"
+          "description": "The user-defined device icon for the frontend. It can be a full URL link to an image (e.g. https://SOME.SITE/MODEL123.jpg) (you cannot use a path to a local file) or base64 encoded data URL (e.g. image/svg+xml;base64,PHN2ZyB3aW....R0aD)"
+        },
+        "homeassistant": {
+          "type": ["object", "null"],
+          "title": "Home Assistant",
+          "properties": {
+              "name": {
+                  "type": "string",
+                  "title": "Home Assistant name",
+                  "description": "Name of the device in Home Assistant"
+              }
+          }
         }
       },
       "required": ["friendly_name"]
@@ -852,6 +897,14 @@
         "qos": {
           "type": "number"
         },
+        "off_state": {
+          "type": ["string"],
+          "enum": ["all_members_off", "last_member_state"],
+          "title": "Group off state",
+          "default": "auto",
+          "requiresRestart": true,
+          "description": "Control when to publish state OFF for a group. 'all_members_off': only publish state OFF when all group memebers are in state OFF, 'last_member_state': publish state OFF whenever one of its members changes to OFF"
+        },
         "filtered_attributes": {
           "type": "array",
           "items": {

lib/util/settings.ts

@@ -3,7 +3,7 @@ import utils from './utils';
 import objectAssignDeep from 'object-assign-deep';
 import path from 'path';
 import yaml from './yaml';
-import Ajv from 'ajv';
+import Ajv, {ValidateFunction} from 'ajv';
 import schemaJson from './settings.schema.json';
 export let schema = schemaJson;
 // @ts-ignore
@@ -31,7 +31,10 @@ const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml
 const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schemaJson);
 const ajvRestartRequired = new Ajv({allErrors: true})
     .addKeyword({keyword: 'requiresRestart', validate: (s: unknown) => !s}).compile(schemaJson);
-
+const ajvRestartRequiredDeviceOptions = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (s: unknown) => !s}).compile(schemaJson.definitions.device);
+const ajvRestartRequiredGroupOptions = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (s: unknown) => !s}).compile(schemaJson.definitions.group);
 const defaults: RecursivePartial<Settings> = {
     permit_join: false,
     external_converters: [],
@@ -72,6 +75,7 @@ const defaults: RecursivePartial<Settings> = {
     device_options: {},
     advanced: {
         legacy_api: true,
+        legacy_availability_payload: true,
         log_rotation: true,
         log_symlink_current: false,
         log_output: ['console', 'file'],
@@ -138,7 +142,7 @@ function loadSettingsWithDefaults(): void {
     }
 
     if (_settingsWithDefaults.availability || _settingsWithDefaults.advanced?.availability_timeout) {
-        const defaults = {active: {timeout: 10}, passive: {timeout: 1500}};
+        const defaults = {};
         const s = typeof _settingsWithDefaults.availability === 'object' ? _settingsWithDefaults.availability : {};
         // @ts-ignore
         _settingsWithDefaults.availability = {};
@@ -186,6 +190,20 @@ function loadSettingsWithDefaults(): void {
     _settingsWithDefaults.whitelist && _settingsWithDefaults.passlist.push(..._settingsWithDefaults.whitelist);
 }
 
+function parseValueRef(text: string): {filename: string, key: string} | null {
+    const match = /!(.*) (.*)/g.exec(text);
+    if (match) {
+        let filename = match[1];
+        // This is mainly for backward compatibility.
+        if (!filename.endsWith('.yaml') && !filename.endsWith('.yml')) {
+            filename += '.yaml';
+        }
+        return {filename, key: match[2]};
+    } else {
+        return null;
+    }
+}
+
 function write(): void {
     const settings = getInternalSettings();
     const toWrite: KeyValue = objectAssignDeep({}, settings);
@@ -195,15 +213,16 @@ function write(): void {
 
     // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
     for (const path of [
+        ['mqtt', 'server'],
         ['mqtt', 'user'],
         ['mqtt', 'password'],
         ['advanced', 'network_key'],
         ['frontend', 'auth_token'],
     ]) {
         if (actual[path[0]] && actual[path[0]][path[1]]) {
-            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
-            if (match) {
-                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+            const ref = parseValueRef(actual[path[0]][path[1]]);
+            if (ref) {
+                yaml.updateIfChanged(data.joinPath(ref.filename), ref.key, toWrite[path[0]][path[1]]);
                 toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
             }
         }
@@ -307,26 +326,31 @@ export function validate(): string[] {
 
 function read(): Settings {
     const s = yaml.read(file) as Settings;
+    applyEnvironmentVariables(s);
 
     // Read !secret MQTT username and password if set
     // eslint-disable-next-line
     const interpetValue = (value: any): any => {
-        const re = /!(.*) (.*)/g;
-        const match = re.exec(value);
-        if (match) {
-            const file = data.joinPath(`${match[1]}.yaml`);
-            const key = match[2];
-            return yaml.read(file)[key];
+        const ref = parseValueRef(value);
+        if (ref) {
+            return yaml.read(data.joinPath(ref.filename))[ref.key];
         } else {
             return value;
         }
     };
 
-    if (s.mqtt?.user && s.mqtt?.password) {
+    if (s.mqtt?.user) {
         s.mqtt.user = interpetValue(s.mqtt.user);
+    }
+
+    if (s.mqtt?.password) {
         s.mqtt.password = interpetValue(s.mqtt.password);
     }
 
+    if (s.mqtt?.server) {
+        s.mqtt.server = interpetValue(s.mqtt.server);
+    }
+
     if (s.advanced?.network_key) {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
@@ -372,7 +396,11 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
                         }, settings);
 
                         if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
-                            setting[key] = JSON.parse(process.env[envVariableName]);
+                            try {
+                                setting[key] = JSON.parse(process.env[envVariableName]);
+                            } catch (error) {
+                                setting[key] = process.env[envVariableName];
+                            }
                         } else if (type.indexOf('number') >= 0) {
                             /* eslint-disable-line */ // @ts-ignore
                             setting[key] = process.env[envVariableName] * 1;
@@ -389,7 +417,7 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
 
                 if (typeof obj[key] === 'object' && obj[key]) {
                     const newPath = [...path];
-                    if (key !== 'properties') {
+                    if (key !== 'properties' && key !== 'oneOf' && !Number.isInteger(Number(key))) {
                         newPath.push(key);
                     }
                     iterate(obj[key], newPath);
@@ -403,7 +431,6 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
 function getInternalSettings(): Partial<Settings> {
     if (!_settings) {
         _settings = read();
-        applyEnvironmentVariables(_settings);
     }
 
     return _settings;
@@ -643,21 +670,27 @@ export function removeGroup(IDorName: string | number): void {
     write();
 }
 
-export function changeEntityOptions(IDorName: string, newOptions: KeyValue): void {
+export function changeEntityOptions(IDorName: string, newOptions: KeyValue): boolean {
     const settings = getInternalSettings();
     delete newOptions.friendly_name;
     delete newOptions.devices;
+    let validator: ValidateFunction;
     if (getDevice(IDorName)) {
         objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
         utils.removeNullPropertiesFromObject(settings.devices[getDevice(IDorName).ID]);
+        validator = ajvRestartRequiredDeviceOptions;
     } else if (getGroup(IDorName)) {
         objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
         utils.removeNullPropertiesFromObject(settings.groups[getGroup(IDorName).ID]);
+        validator = ajvRestartRequiredGroupOptions;
     } else {
         throw new Error(`Device or group '${IDorName}' does not exist`);
     }
 
     write();
+    validator(newOptions);
+    const restartRequired = validator.errors && !!validator.errors.find((e) => e.keyword === 'requiresRestart');
+    return restartRequired;
 }
 
 export function changeFriendlyName(IDorName: string, newName: string): void {

lib/util/utils.ts

@@ -4,6 +4,8 @@ import data from './data';
 import vm from 'vm';
 import fs from 'fs';
 import path from 'path';
+import {detailedDiff} from 'deep-object-diff';
+import objectAssignDeep from 'object-assign-deep';
 
 // construct a local ISO8601 string (instead of UTC-based)
 // Example:
@@ -38,7 +40,7 @@ const endpointNames = [
     'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
     'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
     'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
-    'heat', 'cool', 'water', 'meter', 'wifi',
+    'heat', 'cool', 'water', 'meter', 'wifi', 'no_occupancy_since',
 ];
 
 function capitalize(s: string): string {
@@ -77,7 +79,7 @@ async function getZigbee2MQTTVersion(includeCommitHash=true): Promise<{commitHas
 }
 
 async function getDependencyVersion(depend: string): Promise<{version: string}> {
-    const packageJSON = await import(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
+    const packageJSON = await import(path.join(require.resolve(depend), '..', '..', 'package.json'));
     const version = packageJSON.version;
     return {version};
 }
@@ -132,8 +134,8 @@ function parseJSON(value: string, fallback: string): KeyValue | string {
     }
 }
 
-function loadModuleFromText(moduleCode: string): unknown {
-    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+function loadModuleFromText(moduleCode: string, name?: string): unknown {
+    const moduleFakePath = path.join(__dirname, '..', '..', 'data', 'extension', name || 'externally-loaded.js');
     const sandbox = {
         require: require,
         module: {},
@@ -210,6 +212,43 @@ function toSnakeCase(value: string | KeyValue): any {
     }
 }
 
+function charRange(start: string, stop: string): number[] {
+    const result = [];
+    for (let idx=start.charCodeAt(0), end=stop.charCodeAt(0); idx <=end; ++idx) {
+        result.push(idx);
+    }
+    return result;
+}
+
+const controlCharacters = [
+    ...charRange('\u0000', '\u001F'),
+    ...charRange('\u007f', '\u009F'),
+    ...charRange('\ufdd0', '\ufdef'),
+];
+
+function containsControlCharacter(str: string): boolean {
+    for (let i = 0; i < str.length; i++) {
+        const ch = str.charCodeAt(i);
+        if (controlCharacters.includes(ch) || [0xFFFE, 0xFFFF].includes(ch & 0xFFFF)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+function getAllFiles(path_: string): string[] {
+    const result = [];
+    for (let item of fs.readdirSync(path_)) {
+        item = path.join(path_, item);
+        if (fs.lstatSync(item).isFile()) {
+            result.push(item);
+        } else {
+            result.push(...getAllFiles(item));
+        }
+    }
+    return result;
+}
+
 function validateFriendlyName(name: string, throwFirstError=false): string[] {
     const errors = [];
     for (const endpointName of endpointNames) {
@@ -220,7 +259,7 @@ function validateFriendlyName(name: string, throwFirstError=false): string[] {
 
     if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
     if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
-    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
+    if (containsControlCharacter(name)) errors.push(`friendly_name is not allowed to contain control char`);
     if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
     if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
     if (name.includes('#') || name.includes('+')) {
@@ -245,22 +284,28 @@ function sanitizeImageParameter(parameter: string): string {
     return sanitized;
 }
 
-function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boolean {
-    if (device.options.hasOwnProperty('availability')) {
-        return !!device.options.availability;
+function isAvailabilityEnabledForEntity(entity: Device | Group, settings: Settings): boolean {
+    if (entity.isGroup()) {
+        return !entity.membersDevices().map((d) => isAvailabilityEnabledForEntity(d, settings)).includes(false);
+    }
+
+    if (entity.options.hasOwnProperty('availability')) {
+        return !!entity.options.availability;
     }
 
     // availability_timeout = deprecated
     const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
     if (!enabledGlobal) return false;
 
+    if (entity.isDevice() && entity.options.disabled) return false;
+
     const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
     if (passlist.length > 0) {
-        return passlist.includes(device.name) || passlist.includes(device.ieeeAddr);
+        return passlist.includes(entity.name) || passlist.includes(entity.ieeeAddr);
     }
 
     const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
-    return !blocklist.includes(device.name) && !blocklist.includes(device.ieeeAddr);
+    return !blocklist.includes(entity.name) && !blocklist.includes(entity.ieeeAddr);
 }
 
 const entityIDRegex = new RegExp(`^(.+?)(?:/(${endpointNames.join('|')}|\\d+))?$`);
@@ -273,10 +318,22 @@ function isEndpoint(obj: unknown): obj is zh.Endpoint {
     return obj.constructor.name.toLowerCase() === 'endpoint';
 }
 
+function flatten<Type>(arr: Type[][]): Type[] {
+    return [].concat(...arr);
+}
+
+function arrayUnique<Type>(arr: Type[]): Type[] {
+    return [...new Set(arr)];
+}
+
 function isZHGroup(obj: unknown): obj is zh.Group {
     return obj.constructor.name.toLowerCase() === 'group';
 }
 
+function availabilityPayload(state: 'online' | 'offline', settings: Settings): string {
+    return settings.advanced.legacy_availability_payload ? state : JSON.stringify({state});
+}
+
 const hours = (hours: number): number => 1000 * 60 * 60 * hours;
 const minutes = (minutes: number): number => 1000 * 60 * minutes;
 const seconds = (seconds: number): number => 1000 * seconds;
@@ -296,11 +353,58 @@ function publishLastSeen(data: eventdata.LastSeenChanged, settings: Settings, al
     }
 }
 
+function filterProperties(filter: string[], data: KeyValue): void {
+    if (filter) {
+        for (const property of Object.keys(data)) {
+            if (filter.find((p) => property.match(`^${p}$`))) {
+                delete data[property];
+            }
+        }
+    }
+}
+
+function clone(obj: KeyValue): KeyValue {
+    return JSON.parse(JSON.stringify(obj));
+}
+
+function computeSettingsToChange(current: KeyValue, new_: KeyValue): KeyValue {
+    const diff: KeyValue = detailedDiff(current, new_);
+
+    // Remove any settings that are in the deleted.diff but not in the passed options
+    const cleanupDeleted = (options: KeyValue, deleted: KeyValue): void => {
+        for (const key of Object.keys(deleted)) {
+            if (!(key in options)) {
+                delete deleted[key];
+            } else if (!Array.isArray(options[key])) {
+                cleanupDeleted(options[key], deleted[key]);
+            }
+        }
+    };
+    cleanupDeleted(new_, diff.deleted);
+
+    // objectAssignDeep requires object prototype which is missing from detailedDiff, therefore clone
+    const newSettings = objectAssignDeep({}, clone(diff.added), clone(diff.updated), clone(diff.deleted));
+
+    // deep-object-diff converts arrays to objects, set original array back here
+    const convertBackArray = (before: KeyValue, after: KeyValue): void => {
+        for (const [key, afterValue] of Object.entries(after)) {
+            const beforeValue = before[key];
+            if (Array.isArray(beforeValue)) {
+                after[key] = beforeValue;
+            } else if (afterValue && typeof beforeValue === 'object') {
+                convertBackArray(beforeValue, afterValue);
+            }
+        }
+    };
+    convertBackArray(new_, newSettings);
+    return newSettings;
+}
 
 export default {
     endpointNames, capitalize, getZigbee2MQTTVersion, getDependencyVersion, formatDate, objectHasProperties,
     equalsPartial, getObjectProperty, getResponse, parseJSON, loadModuleFromText, loadModuleFromFile,
     getExternalConvertersDefinitions, removeNullPropertiesFromObject, toNetworkAddressHex, toSnakeCase,
     parseEntityID, isEndpoint, isZHGroup, hours, minutes, seconds, validateFriendlyName, sleep,
-    sanitizeImageParameter, isAvailabilityEnabledForDevice, publishLastSeen,
+    sanitizeImageParameter, isAvailabilityEnabledForEntity, publishLastSeen, availabilityPayload,
+    getAllFiles, filterProperties, flatten, arrayUnique, clone, computeSettingsToChange,
 };

lib/zigbee.ts

@@ -183,6 +183,10 @@ export default class Zigbee {
         return this.herdsman.isStopping();
     }
 
+    async backup(): Promise<void> {
+        return this.herdsman.backup();
+    }
+
     async getNetworkParameters(): Promise<zh.NetworkParameters> {
         return this.herdsman.getNetworkParameters();
     }
@@ -219,7 +223,7 @@ export default class Zigbee {
         }
     }
 
-    private resolveDevice(ieeeAddr: string): Device {
+    @bind private resolveDevice(ieeeAddr: string): Device {
         if (!this.deviceLookup[ieeeAddr]) {
             const device = this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
             device && (this.deviceLookup[ieeeAddr] = new Device(device));
@@ -235,7 +239,7 @@ export default class Zigbee {
     private resolveGroup(groupID: number): Group {
         const group = this.herdsman.getGroupByID(Number(groupID));
         if (group && !this.groupLookup[groupID]) {
-            this.groupLookup[groupID] = new Group(group);
+            this.groupLookup[groupID] = new Group(group, this.resolveDevice);
         }
 
         return this.groupLookup[groupID];
@@ -306,6 +310,10 @@ export default class Zigbee {
         return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
     }
 
+    async addInstallCode(installCode: string): Promise<void> {
+        await this.herdsman.addInstallCode(installCode);
+    }
+
     async touchlinkIdentify(ieeeAddr: string, channel: number): Promise<void> {
         await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.23.0",
+  "version": "1.29.1",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^14 || ^15 || ^16"
+    "node": "^14 || ^16 || ^18 || ^19"
   },
   "keywords": [
     "xiaomi",
@@ -35,55 +35,56 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "^8.9.0",
+    "ajv": "^8.11.2",
     "bind-decorator": "^1.0.11",
-    "connect-gzip-static": "^2.1.1",
-    "core-js": "^3.20.3",
+    "connect-gzip-static": "2.1.1",
     "debounce": "^1.2.1",
-    "deep-object-diff": "^1.1.7",
+    "deep-object-diff": "^1.1.9",
     "fast-deep-equal": "^3.1.3",
-    "finalhandler": "^1.1.2",
+    "finalhandler": "^1.2.0",
     "git-last-commit": "^1.0.1",
-    "humanize-duration": "^3.27.1",
+    "humanize-duration": "^3.27.3",
     "js-yaml": "^4.1.0",
     "json-stable-stringify-without-jsonify": "^1.0.1",
+    "jszip": "^3.10.1",
     "mkdir-recursive": "^0.4.0",
-    "moment": "^2.29.1",
-    "mqtt": "4.3.4",
+    "moment": "^2.29.4",
+    "mqtt": "4.3.7",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
-    "semver": "^7.3.5",
+    "semver": "^7.3.8",
     "source-map-support": "^0.5.21",
     "uri-js": "^4.4.1",
-    "winston": "^3.5.0",
-    "winston-syslog": "^2.4.4",
-    "ws": "^8.4.2",
-    "zigbee-herdsman": "0.14.10",
-    "zigbee-herdsman-converters": "14.0.406",
-    "zigbee2mqtt-frontend": "0.6.71"
+    "winston": "^3.8.2",
+    "winston-syslog": "^2.7.0",
+    "winston-transport": "^4.5.0",
+    "ws": "^8.11.0",
+    "zigbee-herdsman": "0.14.83",
+    "zigbee-herdsman-converters": "15.0.12-hotfix.0",
+    "zigbee2mqtt-frontend": "0.6.121"
   },
   "devDependencies": {
-    "@babel/core": "^7.16.12",
-    "@babel/plugin-proposal-decorators": "^7.16.7",
-    "@babel/preset-env": "^7.16.11",
-    "@babel/preset-typescript": "^7.16.7",
+    "@babel/core": "^7.20.7",
+    "@babel/plugin-proposal-decorators": "^7.20.7",
+    "@babel/preset-env": "^7.20.2",
+    "@babel/preset-typescript": "^7.18.6",
     "@types/debounce": "^1.2.1",
-    "@types/finalhandler": "^1.1.1",
+    "@types/finalhandler": "^1.2.0",
     "@types/humanize-duration": "^3.27.1",
-    "@types/jest": "^27.4.0",
+    "@types/jest": "^29.2.4",
     "@types/js-yaml": "^4.0.5",
     "@types/object-assign-deep": "^0.4.0",
     "@types/rimraf": "^3.0.2",
-    "@types/ws": "^8.2.2",
-    "@typescript-eslint/eslint-plugin": "^5.10.1",
-    "@typescript-eslint/parser": "^5.10.1",
-    "babel-jest": "^27.4.6",
-    "eslint": "^8.8.0",
+    "@types/ws": "^8.5.3",
+    "@typescript-eslint/eslint-plugin": "^5.47.0",
+    "@typescript-eslint/parser": "^5.47.0",
+    "babel-jest": "^29.3.1",
+    "eslint": "^8.30.0",
     "eslint-config-google": "^0.14.0",
-    "eslint-plugin-jest": "^26.0.0",
-    "jest": "^27.4.7",
+    "eslint-plugin-jest": "^27.1.7",
+    "jest": "^29.3.1",
     "tmp": "^0.2.1",
-    "typescript": "^4.5.5"
+    "typescript": "^4.9.4"
   },
   "jest": {
     "coverageThreshold": {

scripts/generateNewSupportedDevices.js

@@ -5,6 +5,8 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
 
 for (const line of text.split('\n')) {
-    const model = zigbeeHerdsmanConverters.devices.find((d) => d.model === line);
-    console.log(`- \`${line}\` ${model.vendor} ${model.description}`);
+    const [modelNumber, user] = line.split(',').map((t) => t.trim());
+    const model = zigbeeHerdsmanConverters.devices.find((d) => d.model === modelNumber);
+    if (!model) throw new Error(`${line} does not exist`);
+    console.log(`- \`${modelNumber}\` ${model.vendor} ${model.description} (${user})`);
 }

scripts/testExternalConverter.js

@@ -0,0 +1,22 @@
+const assert = require('assert');
+const vm = require('vm');
+const fs = require('fs');
+const path = require('path');
+const filename = process.argv[2];
+const moduleCode = fs.readFileSync(filename);
+const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+const sandbox = {
+    require: require,
+    module: {},
+    console,
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
+    setImmediate,
+    clearImmediate,
+};
+vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+const converter = sandbox.module.exports;
+assert(!converter.toZigbee.includes(undefined));
+assert(!converter.fromZigbee.includes(undefined));

test/assets/certs/dummy.crt

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIUU70Q1tbGP071yeQ8EcHd/0rxtnwwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjEyMDMwMDA1MTBaFw0yMzAx
+MDIwMDA1MTBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQDR6bYQPgkfWlGQq19U2nMsAo0RZR7wSUuFS0MFpbQS
+2Rr5x/4FwwKhTMOlfr6H2Xo/5pQvGOlhFE69j1HxQRooWDAswnscMv/9pb+BUqoE
+7J29SEndIpmCaI0c3ejZsu4Nq8DxGod/RsQhJdbDU+hkw8qARJ0JjnTb+aN1KH8g
+J3KeSMTOL3guBVZQujO0Unbjv47yBIFwi0IihR18aoRaxinb94Aqf92HcjZN++WU
+gq7/2Qlm3tF5uJwuNtG/DmzReSblRcUgzPri7qIhb9z51gFNdNX6cTH3yNoF/KtT
+XwEArvvsAyMCYuC6w8KI7WNmPI6yxSksgHdcNPlUVZYquLD+z2kYAdIByiyHmIpm
+OH3jVN7lRJ6xITOGmyNktfRuSUi2GzECAOWIGpDD/5/gK1pacbOXpJWqVrTjUIAN
+BrQf496CbabVgwaQEvq1QJJbuMbpma21/IlS6NpZS7vm/q1ojNsbEB6PYkSsBTzP
+iZiiaomjhV2w5yIp5Zn8xyMFVKLb+SEDQBtZZ6CErmbgqp4B3U2b4k5LippLmZZK
+pSk7BOuMpsZK8FIGS4KlANFfGVs79y6N+Jxue2IO9pRcpcpbqOXty4662BsPKInw
+dKxKTr6TcTe0gEq1fpf1npYwcZ9t36LMJ7L51TbaaJ5sEsXe5QNOOOUgSwPqWmrE
+lwIDAQABo1MwUTAdBgNVHQ4EFgQUYD/x9cCuGJgLUPkYLIVa8uvseBkwHwYDVR0j
+BBgwFoAUYD/x9cCuGJgLUPkYLIVa8uvseBkwDwYDVR0TAQH/BAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAgEAI+gp+8qqO4anj8NHgjJvXhrETLPJIeLuBp/cjF9HA2IL
+oAd5rXWPnQ6qgh3q/HazILgIHwI0wSrhVtRlprgGIxNa59cD3RcZCtqFSWIqhAqZ
+JnAye3bnGU/KahazQX3oP+y564b/QZvwpJDKC1EoMv1wdpXND2NVn4HwaBc7Tu/d
+SX/Tr6k+e32K8mwE+00/yf2rLrmWmNjWtP0f7zRmH3j1+ovktk7UoVhjKXfhqMUE
+pWdhPZIXQJ65JNMGKdeFI2ULQY9IbXHeWQUgBCyhTciMvm2XZzopk7rxrh9NZopS
+pAlCrWBkOZrs6hfZrus9FOzwSkeIEY7NnhNk05kslERJxjkoFytp7mDd4mTKCVW0
+dKc7G1hZWYfO3oe0Zjxi0ragyReNKtyL1BqMbDKJmxHl91iPbdueWix1xxhNFakH
+l3eOzfa/DEixvzxy3hzCcvagNb5RsXFVMQXQTgnIW/5GcG5VVB2/o7KidPL3XtqR
+Hc/0ebQvloGXgqJbyTXG9EInkwB35wXDQuHgwt1PhwzP1yfp2BCvDsQWrR3IxnBT
+DO1o2RpsurgCgUpd0vcW17DePLNfprrhhU2uTPn2bNcWdT/rZmEImkNaA29Rku+3
+SZQJZWIx4s2FMdVdAv6ZauROStTY/WayudfcFfP+hYvJ5/ccQ9Gkb7vrGUssFRw=
+-----END CERTIFICATE-----

test/assets/certs/dummy.key

@@ -0,0 +1,54 @@
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIJnDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQIrNH7wPLdpbUCAggA
+MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECN97bYsRcF6xBIIJSLVWpqTzdlNO
+5J7kOrNcdbbg+SSEYbtT+G9gtwtkI8KO+xzYQZ/fdqmXgNqf9AQmikwDWQkoS4gC
+qq6S+iKYn7Y8ZNZ/xu7gWW6L6uNZ3LTe/vkpenU6kmZrDysFTW5b7iH2VmMc/2h/
+J3o3unxeMLMVSaoDG3MG08qj53yTmetTkZZZc4fgf6vK9/4fTJhdXu6n+1COBLBI
+KbqbTRKnai3CNiYzz3UHyp9CGx1nX06beTJW8bYJUld9cJz+ozLNSV78oCkcUfwG
+yR7p/6+86cWor5SwYG1wEm6w2bXrIJILTUdT92gXC8fhc/B6Id8hcuLIHcvFQ/CP
+31y4Qwea3vY6b1TeRbqS4/fYOYajsiZi9ps0Meg55qMJj/UuNW2iLlR7PVsDXmPO
+33X6cpEVyIlNCJzRREotzSoGmcXjQ5Dgd1LkxZPb163NGkRoJT4GcKCtQxYIyIzy
+qyU0YF7I5LPKtNt1IPURHHerW42jM2jFlISxUixgl5Pj3WrmXeEmXq7htItn6X/q
+tVEVUf6srs3aos2Ohx7Ji9PVDNuMGGGKxleCM6fU9ax/gYuk5r6PMYGPsD26xuZ0
+A9WbFFFI6N1V+M3YsJJdxevZRV1NB3RymYGwFoZFDOKLb5DYJmPP3VVN/DsMNPkw
+qaue7rinxBdLi8J6Itusiscg5pCyYQaujXtkK/bi6c1HCld4RvWCDgeU52SWRdJ9
+2Np2629AeBvBfHP1/DsaUsyQPJAgDJk0HXoJyqstoALkRZ6ICoi1833LW/JGIWOv
+xnl3JZ2q12aqb2OWxiBUKspnt2y5C2KpQJdmuEL68lAKLg+fzLi/jhJG1bwzWo79
+I5G+QeV9+kI3vAIYHu331a+Yevs2p84sIumDGFSCSHAaWMmsrizJ4eKl5NnwZHS+
+xUvcW+ISRKNRm4DWAL/nlwBJWMhCz+nsTKYvE1JO/IH2P5xxtD6bTjJ3YkVAHkBZ
+wfApCBxdiQurEiabj06C4OCcNRkqJ/GwLrdxiwTCAt2hd7Dn+ttLo3Fi6iPRae8c
+ZtWsCWaFObZ1QBeu7lKm+ev/Ab2SHtp6QMfrArNsrdIxSH1+Sm4DgfHmTmte2Tse
+KFHy5Rwokst+Y18zjp3auJ3+NTTr9D1x2JCdQ6wkh66Pyd2FBLqRrMI1fHq9t9P+
+r4DQm9I6FbgALm9yJ2QxV4ytwFGsowAye3vF+hfO5OjAScrG5DrxASRXlnp59zpd
++vMLk6rzcfiHzaJzYc/SOU9ubKN93aJmlL0Zgf1gg54oGnmPbEJeqQzeKVBtisro
+FrNUPEFx5QOaqjVipPOpZfpTb0oaUpBdp4RpeI7W3Gb/dsmsRj7ItxvM6QxFdd5e
+XfDZEYaUJRwSG3lHg9URWwSmfRWV81BhEuwvji4rNIEQ32VMsi2WWWPLOYYKGIw/
+yE56xWBw7K78XwVx/b5yoNHBNjVNyjpNZ7jXkx6zdOjbqEpXJP+WfgVkYwlhlvAq
+cCDbrF7jVWFqTV9tZNxlbaFlS8f17vld8DmmJ29Lo+WecsOo3M9n2Z6i79u7Xg2O
+ip0V2Textq8rrKOwC1TLCRoQLypENO/cwdaF5amVkoXYDlY0rQtxf65Xe9bnJLrM
+1G+tG8NZQu/bvW0mgrm0XHt2i6yfjn6sXEOY5VEC2+gRUriQ+GCETgzMe6ukb26/
+VT38Va10SIu+xXwf/GXTwLskOO0AyWLBopTpL5q6faM4o/USX+Xl98sQVNQurHMS
+xSQtpL6YPhQrmW7q5Q57OVWy70kEU8RpOJx6I7uyoUDz2g3bjmpIPpQX55vhofXK
+3Xn76Ce9TyiVol71mPtBT1tYzi3ytZh/EYm2Wq3lEhbMT7zjw3T1Ypb4+q12VdCy
+0Y9swKsQdhRCT6FwcCFSpCc2TzCCJCR6mChTe+z6rNm5BS2M6BaYcrX8PP82rZ8k
+L0l3ymSLqSJ3J6Q+fdKhRq8EbCq6ngi31Q/l7fPHkW5VAJyowPRayD7UM09n6ou+
+sg/kfQB8nh46uVxRRDpyUsrfhtIEQec2xrU17NmCr2VlAeOjWBn8lE/J1Gfn8WyV
+05mY1aDGcRk/x02+74UujyuozYMfIVDXHpxp8YteTnBfiKVrRq+Aa9zVX9bkyagp
+SJUTpF7lqMmiuvyZnG2xfLDfILR/zTRIRrm+ZIJLHeVQJtibOJ1llxE6kRvKx6zp
+La47YCHKH+rLDTlNgrE5K/JtQgOKc+7bGQ6SCszf8OZWi5B2+Z+Ht36c3xQk9IOW
+cjNpvjZxpiw6L1MhlQPE/wxz2JkOlKjJN0MmLscIyu15X+W/BvHwB8d11CRKUUiE
+pV7OUoW2Ol1IgFHrEY33IhxwtyKtAKfQCiORarnwk/GvcD3TqmnzUeQdscdww4BI
+wsO3c3aGr+wvU3P/8oFAoffD1KRihtoZ6CB347JcnaeEPlOS13a28JsJCOtjMxy9
+wBH6UgzTwo04lDAStST+aIT4TaU4dlWmsAEQTpE8sa4DjpFtzK/ll+Wg2sA/SYaA
+uAugsAOnTb1pkgk43GoYfc2fZCu1g9b/L3mZrE62W8HDfm67PWT65XMJziy/hcCy
+DbXMAWfohLdePKDG7J0QQI8xcTN9TT5q3ZCglI9Xf7PE/qrHROdRgF8c8zyvB9dG
+AVx96Z7akBoveqlFS44MnxIOGI0AzV8MrVbN51W6RHNyg/6UTL50/8zHmRWZj+iO
+tfqiW+CtSHacYPy+Tcuft30PWZZkQDGphbRxaBRB00Fb+Bo57epDw4JS+4wRru33
+2RHaVzQmnXwR6kP85x4uAR53UdgMgfK4cyylnFqGDlLFrHZfauFHg3kMIwqgVIVA
+uMRrWC9XXU11j6NnP9J6Y64uSrjsh6eZOrtJIwA16U2r3PMMbn1hIdADrKu1MsPt
+bkNsnPQUKJyX8DbRX45kje5ssUerbkxhvewvHQJlTyDxtgk/pGHOyKXoT+5vQEj9
+pwHOrXi+QWidzi2vfRoNQJ0wXJr39dDTn0hVdsgFo2PsTh98W9cocVQfJnCL4Djs
+Zlsp/BJOwlb2e3Q28E6pNyT+N63ZYEO9zLV/WmCB/eV2uMVoKibX3KszYkn0jAs0
+RBytgnB8YW3n9RyorShchCKJzdjzR7nw79/lSLgDFE6W6Xh3VaGM07hz0vYWfaui
+yvytkM9LPan2/GJM4zLFeA==
+-----END ENCRYPTED PRIVATE KEY-----

test/availability.test.js

@@ -179,6 +179,15 @@ describe('Availability', () => {
         expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
     });
 
+    it('Should not ping disabled devices', async () => {
+        settings.set(['devices', devices.bulb_color.ieeeAddr, 'disabled'], true);
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(15));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
     it('Should allow to change availability timeout via avaiability options', async () => {
         settings.set(['availability'], {active: {timeout: 30}});
         await resetExtension();
@@ -227,9 +236,23 @@ describe('Availability', () => {
         expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
     });
 
+    it('Should to enable availabilty for just one device', async () => {
+        settings.set(['availability'], false);
+        settings.set(['devices', devices.bulb_color.ieeeAddr, 'availability'], true);
+
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(11));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
     it('Should retrieve device state when it reconnects', async () => {
         MQTT.publish.mockClear();
 
+        const device = controller.zigbee.resolveEntity(devices.bulb_color.ieeeAddr);
+        controller.state.set(device, {state: 'OFF'})
+
         const endpoint = devices.bulb_color.getEndpoint(1);
         endpoint.read.mockClear();
 
@@ -242,18 +265,15 @@ describe('Availability', () => {
         expect(endpoint.read).toHaveBeenCalledTimes(0);
         await advancedTime(utils.seconds(2));
 
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl',
-            ['colorMode', 'currentX', 'currentY', 'enhancedCurrentHue', 'currentSaturation', 'colorTemperature']);
 
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
         await flushPromises();
         endpoint.read.mockImplementationOnce(() => {throw new Error('')});
         await advancedTime(utils.seconds(3));
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
     });
 
     it('Should republish availability when device is renamed', async () => {
@@ -271,6 +291,16 @@ describe('Availability', () => {
             'offline', {retain: true, qos: 0}, expect.any(Function));
     });
 
+    it('Should publish availabiltiy payload in JSON format', async () => {
+        settings.set(['advanced', 'legacy_availability_payload'], false);
+        await resetExtension();
+        MQTT.publish.mockClear();
+        await advancedTime(utils.hours(26));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            stringify({state: 'offline'}), {retain: true, qos: 0}, expect.any(Function));
+    });
+
     it('Deprecated - should allow to block via advanced.availability_blocklist', async () => {
         settings.set(['advanced', 'availability_blocklist'], [devices.bulb_color.ieeeAddr]);
         await resetExtension();
@@ -297,4 +327,21 @@ describe('Availability', () => {
         await advancedTime(utils.minutes(12));
         expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
     });
+
+    it('Should publish availabilty for groups', async () => {
+        settings.set(['devices', devices.bulb_color_2.ieeeAddr, 'availability'], true);
+        await resetExtension();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        MQTT.publish.mockClear();
+        await advancedTime(utils.minutes(12));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+        MQTT.publish.mockClear();
+        devices.bulb_color_2.lastSeen = Date.now();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color_2});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
 });

test/bind.test.js

@@ -588,6 +588,7 @@ describe('Bind', () => {
     it('Should poll bounded Hue bulb when receiving message from scene controller', async () => {
         const remote = zigbeeHerdsman.devices.bj_scene_switch;
         const data = {"action": "recall_2_row_1"};
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockImplementationOnce(() => {throw new Error('failed')});
         const payload = {data, cluster: 'genScenes', device: remote, endpoint: remote.getEndpoint(10), type: 'commandRecall', linkquality: 10, groupID: 0};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
@@ -595,7 +596,7 @@ describe('Bind', () => {
         expect(debounce).toHaveBeenCalledTimes(3);
         expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
         expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
-	expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("lightingColorCtrl", ["currentX", "currentY", "colorTemperature"]);
+	    expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("lightingColorCtrl", ["currentX", "currentY", "colorTemperature"]);
     });
 
     it('Should poll grouped Hue bulb when receiving message from TRADFRI remote', async () => {

test/configure.test.js

@@ -65,6 +65,7 @@ describe('Configure', () => {
         jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        await jest.runOnlyPendingTimers();
         await flushPromises();
     });
 
@@ -72,8 +73,9 @@ describe('Configure', () => {
         data.writeDefaultConfiguration();
         settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
-     coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         await resetExtension();
+        await jest.runOnlyPendingTimers();
     });
 
     afterAll(async () => {
@@ -100,6 +102,17 @@ describe('Configure', () => {
         expectBulbConfigured();
     });
 
+    it('Should not re-configure disabled devices', async () => {
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
+        await flushPromises();
+        mockClear(device);
+        settings.set(['devices', device.ieeeAddr, 'disabled'], true);
+        zigbeeHerdsman.events.deviceJoined({device});
+        await flushPromises();
+        expectBulbNotConfigured();
+    });
+
     it('Should reconfigure reporting on reconfigure event', async () => {
         expectBulbConfigured();
         const device = controller.zigbee.resolveEntity(zigbeeHerdsman.devices.bulb);
@@ -192,19 +205,7 @@ describe('Configure', () => {
         expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
     });
 
-    it('Should not configure when interviewing', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        delete device.meta.configured;
-        device.interviewing = true;
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        await zigbeeHerdsman.events.lastSeenChanged({device});
-        await flushPromises();
-        expectRemoteNotConfigured();
-        device.interviewing = false;
-    });
-
-    it('Should configure when not interviewCompleted', async () => {
+    it('Should not configure when interview not completed', async () => {
         const device = zigbeeHerdsman.devices.remote;
         delete device.meta.configured;
         device.interviewCompleted = false;
@@ -212,7 +213,7 @@ describe('Configure', () => {
         mockClear(device);
         await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
-        expectRemoteConfigured();
+        expectRemoteNotConfigured();
         device.interviewCompleted = true;
     });
 

test/controller.test.js

@@ -120,6 +120,7 @@ describe('Controller', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", stringify({"brightness":255}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":'ON'}), {"qos": 0, "retain":false}, expect.any(Function));
     });
 
     it('Start controller should not publish cached states when disabled', async () => {
@@ -201,9 +202,10 @@ describe('Controller', () => {
         });
         await controller.start();
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith('MQTT failed to connect: addr not found');
+        expect(logger.error).toHaveBeenCalledWith('MQTT error: addr not found');
+        expect(logger.error).toHaveBeenCalledWith('MQTT failed to connect, exiting...');
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, false);
     });
 
     it('Start controller with permit join true', async () => {
@@ -220,13 +222,13 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledWith(14);
     });
 
-    it('Start controller and stop', async () => {
+    it('Start controller and stop with restart', async () => {
         await controller.start();
-        await controller.stop();
+        await controller.stop(true);
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(0);
+        expect(mockExit).toHaveBeenCalledWith(0, true);
     });
 
     it('Start controller and stop', async () => {
@@ -236,7 +238,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, false);
     });
 
     it('Start controller adapter disconnects', async () => {
@@ -247,7 +249,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, false);
     });
 
     it('Handle mqtt message', async () => {
@@ -489,6 +491,46 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
+    it('Publish entity state attribute_json output filtered cache', async () => {
+        await controller.start();
+        settings.set(['advanced', 'output'], 'attribute_and_json');
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_cache'], ['linkquality']);
+        MQTT.publish.mockClear();
+
+        const device = controller.zigbee.resolveEntity('bulb');
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
+
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
+        await flushPromises();
+
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output filtered cache (device_options)', async () => {
+        await controller.start();
+        settings.set(['advanced', 'output'], 'attribute_and_json');
+        settings.set(['device_options', 'filtered_cache'], ['linkquality']);
+        MQTT.publish.mockClear();
+
+        const device = controller.zigbee.resolveEntity('bulb');
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
+
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
+        await flushPromises();
+
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
     it('Publish entity state with device information', async () => {
         await controller.start();
         settings.set(['mqtt', 'include_device_information'], true);
@@ -569,7 +611,7 @@ describe('Controller', () => {
         logger.error.mockClear();
         controller.state.file = "/";
         await controller.state.save();
-        expect(logger.error).toHaveBeenCalledWith(`Failed to write state to '/' (EISDIR: illegal operation on a directory, open '/')`);
+        expect(logger.error).toHaveBeenCalledWith(expect.stringMatching(/Failed to write state to \'\/\'/));
     });
 
     it('Publish should not cache when set', async () => {
@@ -604,6 +646,27 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should republish retained messages on MQTT reconnect', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        MQTT.events['connect']();
+        await flushPromises();
+        jest.runOnlyPendingTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(13);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Should not republish retained messages on MQTT reconnect when retained message are sent', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        MQTT.events['connect']();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/state', 'online');
+        jest.runOnlyPendingTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/state', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
     it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
         settings.set(['mqtt', 'force_disable_retain'], true);
         await controller.mqtt.connect()
@@ -656,4 +719,13 @@ describe('Controller', () => {
         await flushPromises();
         expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'Coordinator', type 'attributeReport', cluster 'genBasic', data '{"modelId":null}' from endpoint 1, ignoring since it is from coordinator`);
     });
+
+    it('Should remove state of removed device when stopped', async () => {
+        await controller.start();
+        const device = controller.zigbee.resolveEntity('bulb');
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
+        device.zh.isDeleted = true;
+        await controller.stop();
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual(undefined);
+    });
 });

test/data.test.js

@@ -19,6 +19,7 @@ describe('Data', () => {
             const actual = data.getPath();
             expect(actual).toBe(expected);
             expect(data.joinPath('test')).toStrictEqual(path.join(expected, 'test'));
+            expect(data.joinPath('/test')).toStrictEqual(path.resolve(expected, '/test'));
             delete process.env.ZIGBEE2MQTT_DATA;
             data.testingOnlyReload();
         });

test/externalExtension.test.js

@@ -84,7 +84,12 @@ describe('User extensions', () => {
         MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
         await flushPromises();
 
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/response/extension/save', stringify({"data":{},"error":"Unexpected identifier","status":"error"}), { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/response/extension/save', expect.any(String), { retain: false, qos: 0 }, expect.any(Function));
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload).toEqual(
+            expect.objectContaining({"data":{},"status":"error"})
+        );
+        expect(payload.error).toMatch("Unexpected identifier");
     });
 
     it('Removes user extension', async () => {

test/frontend.test.js

@@ -7,6 +7,7 @@ const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
 const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const path = require("path");
 jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mockHTTP = {
@@ -19,6 +20,16 @@ const mockHTTP = {
     events: {},
 };
 
+const mockHTTPS = {
+    implementation: {
+        listen: jest.fn(),
+        on: (event, handler) => {mockHTTPS.events[event] = handler},
+        close: jest.fn().mockImplementation((cb) => cb()),
+    },
+    variables: {},
+    events: {},
+};
+
 const mockWSocket = {
     close: jest.fn(),
 };
@@ -50,6 +61,13 @@ jest.mock('http', () => ({
     }),
 }));
 
+jest.mock('https', () => ({
+    createServer: jest.fn().mockImplementation((onRequest) => {
+        mockHTTPS.variables.onRequest = onRequest;
+        return mockHTTPS.implementation;
+    }),
+}));
+
 jest.mock("connect-gzip-static", () =>
     jest.fn().mockImplementation((path) => {
         mockNodeStatic.variables.path = path
@@ -98,7 +116,6 @@ describe('Frontend', () => {
         await controller.start();
         expect(mockNodeStatic.variables.path).toBe("my/dummy/path");
         expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
-
         const mockWSClient = {
             implementation: {
                 terminate: jest.fn(),
@@ -111,6 +128,46 @@ describe('Frontend', () => {
         expect(mockWSClient.implementation.terminate).toHaveBeenCalledTimes(1);
         expect(mockHTTP.implementation.close).toHaveBeenCalledTimes(1);
         expect(mockWS.implementation.close).toHaveBeenCalledTimes(1);
+        mockHTTP.implementation.listen.mockClear();
+        mockHTTPS.implementation.listen.mockClear();
+    });
+
+
+    it('Start/stop HTTPS valid', async () => {
+        settings.set(['frontend','ssl_cert'], path.join(__dirname,'assets','certs','dummy.crt'));
+        settings.set(['frontend','ssl_key'],  path.join(__dirname,'assets','certs','dummy.key'));
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        expect(mockHTTP.implementation.listen).not.toHaveBeenCalledWith(8081, "127.0.0.1");
+        expect(mockHTTPS.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+        await controller.stop();
+        mockHTTP.implementation.listen.mockClear();
+        mockHTTPS.implementation.listen.mockClear();
+    });
+
+    it('Start/stop HTTPS invalid : missing config', async () => {
+        settings.set(['frontend','ssl_cert'], path.join(__dirname,'assets','certs','dummy.crt'));
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+        expect(mockHTTPS.implementation.listen).not.toHaveBeenCalledWith(8081, "127.0.0.1");
+        await controller.stop();
+        mockHTTP.implementation.listen.mockClear();
+        mockHTTPS.implementation.listen.mockClear();
+
+    });
+
+    it('Start/stop HTTPS invalid : missing file', async () => {
+        settings.set(['frontend','ssl_cert'], 'filesNotExists.crt');
+        settings.set(['frontend','ssl_key'],  path.join(__dirname,'assets','certs','dummy.key'));
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+        expect(mockHTTPS.implementation.listen).not.toHaveBeenCalledWith(8081, "127.0.0.1");
+        await controller.stop();
+        mockHTTP.implementation.listen.mockClear();
+        mockHTTPS.implementation.listen.mockClear();
+
     });
 
     it('Websocket interaction', async () => {
@@ -140,7 +197,7 @@ describe('Frontend', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color',
-            stringify({state: 'ON', linkquality: null, update_available: null, update: {state: null}}),
+             stringify({state: 'ON', power_on_behavior:null, linkquality: null, update_available: null, update: {state: null, installed_version: -1, latest_version: -1}}),
             { retain: false, qos: 0 },
             expect.any(Function)
         );
@@ -151,7 +208,7 @@ describe('Frontend', () => {
 
         // Received message on socket
         expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(1);
-        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', linkquality: null, update_available: null, update: {state: null}}}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', power_on_behavior:null, linkquality: null, update_available: null, update: {state: null, installed_version: -1, latest_version: -1}}}));
 
         // Shouldnt set when not ready
         mockWSClient.implementation.send.mockClear();

test/group.test.js

@@ -334,6 +334,22 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
+    it('Should not publish state change when group changes state and device is disabled', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['devices', device.ieeeAddr, 'disabled'], true);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
     it('Should publish state change for group when members state change', async () => {
         // Created for https://github.com/Koenkk/zigbee2mqtt/issues/5725
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -469,6 +485,30 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
+    it('Should publish state change off if any lights within are still on when changed via device when off_state: last_member_state is used', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false, off_state: 'last_member_state'}
+        });
+        await resetExtension();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1, "zigbee2mqtt/group_1", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2, "zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
         const device_1 = zigbeeHerdsman.devices.bulb_color;
         const device_2 = zigbeeHerdsman.devices.bulb;

test/homeassistant.test.js

@@ -58,7 +58,12 @@ describe('HomeAssistant extension', () => {
             configs.forEach((c) => {
                 const id = c['type'] + '/' + c['object_id'];
                 if (cfg_type_object_ids.includes(id)) {
-                    duplicated.push(d.model);
+                    if (typeof d.exposes == 'function') {
+                        // A dynamic function must exposes all possible attributes for the docs
+                        console.warn(`${d.model} dynamic exposes contains duplicated ${id}`)
+                    } else {
+                        duplicated.push(d.model);
+                    }
                 } else {
                     cfg_type_object_ids.push(id);
                 }
@@ -93,6 +98,15 @@ describe('HomeAssistant extension', () => {
                "xy",
                "color_temp"
             ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
             "unique_id":"9_light_zigbee2mqtt"
         };
 
@@ -740,32 +754,36 @@ describe('HomeAssistant extension', () => {
         let payload;
 
         payload = {
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
-            "away_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/away_mode",
-            "away_mode_state_template":"{{ value_json.away_mode }}",
-            "away_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "action_template":"{% set values = {None:None,'idle':'off','heat':'heating','cool':'cooling','fan_only':'fan'} %}{{ values[value_json.running_state] }}",
+            "action_topic":"zigbee2mqtt/TS0601_thermostat",
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
             "current_temperature_template":"{{ value_json.local_temperature }}",
             "current_temperature_topic":"zigbee2mqtt/TS0601_thermostat",
             "device":{
-               "identifiers":[
-                  "zigbee2mqtt_0x0017882104a44559"
-               ],
-               "manufacturer":"TuYa",
-               "model":"Radiator valve with thermostat (TS0601_thermostat)",
-               "name":"TS0601_thermostat",
-               "sw_version":  null,
+                "identifiers":[
+                    "zigbee2mqtt_0x0017882104a44559"
+                ],
+                "manufacturer":"TuYa",
+                "model":"Radiator valve with thermostat (TS0601_thermostat)",
+                "name":"TS0601_thermostat",
+                "sw_version":null
             },
-            "hold_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
-            "hold_modes":[
-               "schedule",
-               "manual",
-               "boost",
-               "complex",
-               "comfort",
-               "eco"
+            "preset_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
+            "preset_modes":[
+                "schedule",
+                "manual",
+                "boost",
+                "complex",
+                "comfort",
+                "eco",
+                "away"
             ],
-            "hold_state_template":"{{ value_json.preset }}",
-            "hold_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "preset_mode_value_template":"{{ value_json.preset }}",
+            "preset_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "json_attributes_topic":"zigbee2mqtt/TS0601_thermostat",
             "max_temp":"35",
             "min_temp":"5",
@@ -773,7 +791,9 @@ describe('HomeAssistant extension', () => {
             "mode_state_template":"{{ value_json.system_mode }}",
             "mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "modes":[
-               "heat", "auto", "off"
+                "heat",
+                "auto",
+                "off"
             ],
             "name":"TS0601_thermostat",
             "temp_step":0.5,
@@ -801,6 +821,10 @@ describe('HomeAssistant extension', () => {
             set_position_topic: 'zigbee2mqtt/smart vent/set',
             set_position_template: '{ "position": {{ position }} }',
             position_template: '{{ value_json.position }}',
+            state_topic: 'zigbee2mqtt/smart vent',
+            value_template: `{{ value_json.state }}`,
+            state_open: 'OPEN',
+            state_closed: 'CLOSE',
             json_attributes_topic: 'zigbee2mqtt/smart vent',
             name: 'smart vent',
             unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
@@ -821,6 +845,39 @@ describe('HomeAssistant extension', () => {
             { retain: true, qos: 0 },
             expect.any(Function),
         );
+
+        payload = {
+            "availability": [{ "topic": "zigbee2mqtt/bridge/state" }],
+            "command_topic": "zigbee2mqtt/zigfred_plus/l6/set",
+            "device": {
+                "identifiers": ["zigbee2mqtt_0xf4ce368a38be56a1"],
+                "manufacturer": "Siglis",
+                "model": "zigfred plus smart in-wall switch (ZFP-1A-CH)",
+                "name": "zigfred_plus",
+                "sw_version": null
+            },
+            "json_attributes_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "name": "zigfred_plus_l6",
+            "position_template": "{{ value_json.position }}",
+            "position_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "set_position_template": "{ \"position_l6\": {{ position }} }",
+            "set_position_topic": "zigbee2mqtt/zigfred_plus/l6/set",
+            "state_closed": "CLOSE",
+            "state_open": "OPEN",
+            "state_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "tilt_command_topic": "zigbee2mqtt/zigfred_plus/l6/set/tilt",
+            "tilt_status_template": "{{ value_json.tilt }}",
+            "tilt_status_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "unique_id": "0xf4ce368a38be56a1_cover_l6_zigbee2mqtt",
+            "value_template": "{{ value_json.state }}"
+        }
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/cover/0xf4ce368a38be56a1/cover_l6/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
     });
 
     it('Should discover devices with custom homeassistant_discovery_topic', async () => {
@@ -899,7 +956,7 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color',
-            stringify({"color":{"hue": 0, "saturation": 100, "h": 0, "s": 100}, "color_mode": "hs", "linkquality": null, "state": null, "update_available": null, "update": {"state": null}}),
+            stringify({"color":{"hue": 0, "saturation": 100, "h": 0, "s": 100}, "color_mode": "hs", "linkquality": null, "state": null, "update_available": null, "power_on_behavior":null, "update": {"state": null, "installed_version": -1, "latest_version": -1}}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -915,7 +972,7 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color',
-            stringify({"color": {"x": 0.4576,"y": 0.41}, "color_mode": "xy", "linkquality": null,"state": null, "update_available": null, "update": {"state": null}}),
+            stringify({"color": {"x": 0.4576,"y": 0.41}, "color_mode": "xy", "linkquality": null,"state": null, "update_available": null, "power_on_behavior":null, "update": {"state": null, "installed_version": -1, "latest_version": -1}}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -931,7 +988,7 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color',
-            stringify({"linkquality": null,"state": "ON", "update_available": null, "update": {"state": null}}),
+            stringify({"linkquality": null,"state": "ON", "update_available": null, "power_on_behavior": null, "update": {"state": null, "installed_version": -1, "latest_version": -1}}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -993,6 +1050,19 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
     });
 
+    it('Should discover when options change', async () => {
+        const device = controller.zigbee.resolveEntity(zigbeeHerdsman.devices.bulb);
+        MQTT.publish.mockClear();
+        controller.eventBus.emitEntityOptionsChanged({entity: device, from: {}, to: {'test': 123}});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            `homeassistant/light/${device.ID}/light/config`,
+            expect.any(String),
+            expect.any(Object),
+            expect.any(Function),
+        );
+    });
+
     it('Should send all status when home assistant comes online (default topic)', async () => {
         data.writeDefaultState();
         extension.state.load();
@@ -1006,16 +1076,22 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null, "update": {"state": null}}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null, "update": {"state": null, "installed_version": -1, "latest_version": -1}}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"action":null,"action_duration":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null, "update": {"state": null}}),
+            stringify({"action":null,"action_duration":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null, "update": {"state": null, "installed_version": -1, "latest_version": -1}}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/group_1',
+            stringify({"state":'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Should send all status when home assistant comes online', async () => {
@@ -1030,13 +1106,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null, "update": {"state": null}}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null, "update": {"state": null, "installed_version": -1, "latest_version": -1}}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"action":null,"action_duration":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null, "update": {"state": null}}),
+            stringify({"action":null,"action_duration":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null, "update": {"state": null, "installed_version": -1, "latest_version": -1}}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -1152,6 +1228,8 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":true}));
         await flushPromises();
+        jest.runOnlyPendingTimers();
+        await flushPromises();
 
         const payload = {
             'unit_of_measurement': '°C',
@@ -1214,6 +1292,8 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/rename', stringify({"from": "ha_discovery_group", "to": "ha_discovery_group_new","homeassistant_rename":true}));
         await flushPromises();
+        jest.runOnlyPendingTimers();
+        await flushPromises();
 
         const payload = {
             "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
@@ -1236,6 +1316,15 @@ describe('HomeAssistant extension', () => {
                "xy",
                "color_temp"
             ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
             "unique_id":"9_light_zigbee2mqtt"
          };
 
@@ -1299,7 +1388,7 @@ describe('HomeAssistant extension', () => {
             "payload_on":true,
             "payload_off":false,
             "value_template":`{{ value_json['update']['state'] == "available" }}`,
-            "enabled_by_default": true,
+            "enabled_by_default": false,
             "state_topic":"zigbee2mqtt/bulb",
             "json_attributes_topic":"zigbee2mqtt/bulb",
             "name":"bulb update available",
@@ -1328,7 +1417,7 @@ describe('HomeAssistant extension', () => {
 
     it('Should discover trigger when click is published', async () => {
         const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(5);
+        expect(discovered.length).toBe(7);
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/click/config');
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');
 
@@ -1403,21 +1492,21 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: "single", click: "single", battery: null, linkquality: null, voltage: null}),
+            stringify({action: "single", click: "single", battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: "", battery: null, linkquality: null, voltage: null, click: null}),
+            stringify({action: "", battery: null, linkquality: null, voltage: null, click: null, power_outage_count: null, device_temperature: null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({click: "", action: null, battery: null, linkquality: null, voltage: null}),
+            stringify({click: "", action: null, battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -1504,7 +1593,7 @@ describe('HomeAssistant extension', () => {
         await resetExtension();
 
         const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(4);
+        expect(discovered.length).toBe(6);
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/battery/config');
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/linkquality/config');
@@ -1515,7 +1604,7 @@ describe('HomeAssistant extension', () => {
         await resetExtension();
 
         const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(3);
+        expect(discovered.length).toBe(5);
         expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/click/config');
         expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/action/config');
 
@@ -1560,7 +1649,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: "single", "battery":null,"linkquality":null,"voltage":null}),
+            stringify({action: "single", "battery":null,"linkquality":null,"voltage":null, "power_outage_count": null, "device_temperature": null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -1597,10 +1686,10 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', click: null, battery: null, linkquality: null, voltage: null});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', click: null, battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', click: null, battery: null, linkquality: null, voltage: null});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', click: null, battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/action_single/config');
         expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/action');
@@ -1752,6 +1841,15 @@ describe('HomeAssistant extension', () => {
                "xy",
                "color_temp"
             ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
             "unique_id":"9_light_zigbee2mqtt"
          };
 
@@ -1763,6 +1861,105 @@ describe('HomeAssistant extension', () => {
         );
     });
 
+    it('Should discover with json availability payload value_template', async () => {
+        settings.set(['advanced', 'legacy_availability_payload'], false);
+        await resetExtension();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state","value_template":'{{ value_json.state }}'}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version": version,
+            },
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should discover with availability offline when device is disabled', async () => {
+        settings.set(['devices', '0x000b57fffec6a5b2', 'disabled'], true);
+
+        await resetExtension();
+
+        const payload = {
+            "availability":[
+               {
+                  "topic":"zigbee2mqtt/bridge/state",
+                  "value_template": `{{ "offline" }}`,
+               }
+            ],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/bulb/set",
+            "device":{
+               "identifiers":[
+                  "zigbee2mqtt_0x000b57fffec6a5b2"
+               ],
+               "manufacturer":"IKEA",
+               "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
+               "name":"bulb",
+               "sw_version":null
+            },
+            "effect":true,
+            "effect_list":[
+               "blink",
+               "breathe",
+               "okay",
+               "channel_change",
+               "finish_effect",
+               "stop_effect"
+            ],
+            "json_attributes_topic":"zigbee2mqtt/bulb",
+            "max_mireds":454,
+            "min_mireds":250,
+            "name":"bulb",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/bulb",
+            "supported_color_modes":[
+               "color_temp"
+            ],
+            "unique_id":"0x000b57fffec6a5b2_light_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/0x000b57fffec6a5b2/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
     it('Should discover last_seen when enabled', async () => {
         settings.set(['advanced', 'last_seen'], 'ISO_8601');
         await resetExtension();

test/logger.test.js

@@ -25,6 +25,7 @@ describe('Logger', () => {
 
     it('Create log directory', () => {
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
         const dirs = fs.readdirSync(dir.name);
         expect(dirs.length).toBe(1);
@@ -32,6 +33,7 @@ describe('Logger', () => {
 
     it('Should cleanup', () => {
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
 
         for (const d of fs.readdirSync(dir.name)) {
@@ -49,6 +51,7 @@ describe('Logger', () => {
 
     it('Should not cleanup when there is no timestamp set', () => {
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
         for (let i = 30; i < 40; i++) {
             fs.mkdirSync(path.join(dir.name, `log_${i}`));
@@ -62,6 +65,7 @@ describe('Logger', () => {
 
     it('Set and get log level', () => {
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
         logger.setLevel('debug');
         expect(logger.getLevel()).toBe('debug');
@@ -74,23 +78,26 @@ describe('Logger', () => {
         }
 
         const logger = require('../lib/util/logger').default;
-        expect(logger.winston.transports.length).toBe(2);
+        logger.init();
+        expect(logger.winston().transports.length).toBe(2);
         logger.addTransport(new DummyTransport());
-        expect(logger.winston.transports.length).toBe(3);
+        expect(logger.winston().transports.length).toBe(3);
     });
 
     it('Set and get log level warn <-> warning', () => {
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
         logger.setLevel('warn');
-        expect(logger.winston.transports[0].level).toBe('warning');
+        expect(logger.winston().transports[0].level).toBe('warning');
         expect(logger.getLevel()).toBe('warn');
     });
 
     it('Logger should be console and file by default', () => {
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
         expect(pipes[0].silent).toBe(false);
@@ -101,8 +108,9 @@ describe('Logger', () => {
     it('Logger can be file only', () => {
         settings.set(['advanced', 'log_output'], ['file']);
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
         expect(pipes[0].silent).toBe(true);
@@ -113,8 +121,9 @@ describe('Logger', () => {
     it('Logger can be console only', () => {
         settings.set(['advanced', 'log_output'], ['console']);
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(false);
     });
@@ -122,8 +131,9 @@ describe('Logger', () => {
     it('Logger can be nothing', () => {
         settings.set(['advanced', 'log_output'], []);
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(true);
     });
@@ -131,8 +141,9 @@ describe('Logger', () => {
     it('Should allow to disable log rotation', () => {
         settings.set(['advanced', 'log_rotation'], false);
         const logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
         expect(pipes[1].constructor.name).toBe('File');
         expect(pipes[1].maxFiles).toBeNull();
         expect(pipes[1].tailable).toBeFalsy();
@@ -142,6 +153,7 @@ describe('Logger', () => {
     it('Should allow to symlink logs to current directory', () => {
         settings.set(['advanced', 'log_symlink_current'], true);
         let logger = require('../lib/util/logger').default;
+        logger.init();
         logger.logOutput();
         expect(fs.readdirSync(dir.name).includes('current')).toBeTruthy()
 
@@ -151,23 +163,24 @@ describe('Logger', () => {
 
     it('Log', () => {
         const logger = require('../lib/util/logger').default;
-        const warn = jest.spyOn(logger.winston, 'warning');
+        logger.init();
+        const warn = jest.spyOn(logger.winston(), 'warning');
         logger.warn('warn');
         expect(warn).toHaveBeenCalledWith('warn');
 
-        const debug = jest.spyOn(logger.winston, 'debug');
+        const debug = jest.spyOn(logger.winston(), 'debug');
         logger.debug('debug');
         expect(debug).toHaveBeenCalledWith('debug');
 
-        const warning = jest.spyOn(logger.winston, 'warning');
+        const warning = jest.spyOn(logger.winston(), 'warning');
         logger.warning('warning');
         expect(warning).toHaveBeenCalledWith('warning');
 
-        const info = jest.spyOn(logger.winston, 'info');
+        const info = jest.spyOn(logger.winston(), 'info');
         logger.info('info');
         expect(info).toHaveBeenCalledWith('info');
 
-        const error = jest.spyOn(logger.winston, 'error');
+        const error = jest.spyOn(logger.winston(), 'error');
         logger.error('error');
         expect(error).toHaveBeenCalledWith('error');
     });

test/networkMap.test.js

@@ -102,7 +102,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
+        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, power_on_behavior, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, power_outage_count, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -276,7 +276,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
 
-        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, power_on_behavior, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, power_outage_count, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
         const actual = JSON.parse(call[1]);
         expect(actual).toStrictEqual(expected);
     });
@@ -292,4 +292,19 @@ describe('Networkmap', () => {
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
+
+    it('Should exclude disabled devices from networkmap', async () => {
+        settings.set(['devices', '0x000b57fffec6a5b2', 'disabled'], true);
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', stringify({type: 'raw', routes: true}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
+
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, power_on_behavior, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, power_outage_count, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
+        const actual = JSON.parse(call[1]);
+        expect(actual).toStrictEqual(expected);
+    });
 });

test/onEvent.test.js

@@ -48,6 +48,8 @@ describe('On event', () => {
         expect(call[0]).toBe('start')
         expect(call[1]).toStrictEqual({})
         expect(call[2]).toBe(device);
+        expect(call[3]).toStrictEqual(settings.getDevice(device.ieeeAddr));
+        expect(call[4]).toStrictEqual({});
     });
 
     it('Should call with stop event', async () => {
@@ -66,7 +68,7 @@ describe('On event', () => {
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
         expect(mockOnEvent).toHaveBeenCalledTimes(1);
-        expect(mockOnEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr));
+        expect(mockOnEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr), {});
     });
 
     it('Should call index onEvent with zigbee event', async () => {
@@ -74,6 +76,6 @@ describe('On event', () => {
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
         expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device);
+        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr), {});
     });
 });

test/otaUpdate.test.js

@@ -1,3 +1,5 @@
+const path = require('path');
+
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
@@ -11,6 +13,7 @@ const zigbeeOTA = require('zigbee-herdsman-converters/lib/ota/zigbeeOTA');
 
 const spyUseIndexOverride = jest.spyOn(zigbeeOTA, 'useIndexOverride');
 
+
 describe('OTA update', () => {
     let controller;
 
@@ -33,6 +36,7 @@ describe('OTA update', () => {
         jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        await jest.runOnlyPendingTimers();
         await flushPromises();
     });
 
@@ -68,6 +72,7 @@ describe('OTA update', () => {
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             onUpdate(0, null);
             onUpdate(10, 3600.2123);
+            return 90;
         });
 
         MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', 'bulb');
@@ -95,7 +100,7 @@ describe('OTA update', () => {
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"update_available":false,"update":{"state":"idle"}}),
+            stringify({"update_available":false,"update":{"state":"idle","installed_version":90,"latest_version":90}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -141,7 +146,7 @@ describe('OTA update', () => {
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: false, currentFileVersion: 10, otaFileVersion: 10});
         MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
@@ -153,7 +158,7 @@ describe('OTA update', () => {
         );
 
         MQTT.publish.mockClear();
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: true, currentFileVersion: 10, otaFileVersion: 12});
         MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
@@ -246,7 +251,7 @@ describe('OTA update', () => {
         const data = {imageType: 12382};
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: true, currentFileVersion: 10, otaFileVersion: 12});
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
         logger.info.mockClear();
         await zigbeeHerdsman.events.message(payload);
@@ -255,7 +260,7 @@ describe('OTA update', () => {
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
         expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x98});
 
         // Should not request again when device asks again after a short time
         await zigbeeHerdsman.events.message(payload);
@@ -263,13 +268,13 @@ describe('OTA update', () => {
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
 
         logger.info.mockClear();
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: false, currentFileVersion: 10, otaFileVersion: 10});
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"update_available":true,"update":{"state":"available"}}),
+            stringify({"update_available":true,"update":{"state":"available","installed_version":10,"latest_version":12}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
     });
@@ -302,7 +307,7 @@ describe('OTA update', () => {
         const data = {imageType: 12382};
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: false, currentFileVersion: 13, otaFileVersion: 13});
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
         logger.info.mockClear();
         await zigbeeHerdsman.events.message(payload);
@@ -310,10 +315,10 @@ describe('OTA update', () => {
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x98});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"update_available":false,"update":{"state":"idle"}}),
+            stringify({"update_available":false,"update":{"state":"idle","installed_version": 13, "latest_version": 13}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
     });
@@ -324,7 +329,7 @@ describe('OTA update', () => {
         const data = {imageType: 12382};
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: true, currentFileVersion: 10, otaFileVersion: 13});
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
         logger.info.mockClear();
         await zigbeeHerdsman.events.message(payload);
@@ -369,6 +374,7 @@ describe('OTA update', () => {
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             onUpdate(0, null);
             onUpdate(10, 3600);
+            return 91;
         });
 
         MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
@@ -411,7 +417,7 @@ describe('OTA update', () => {
         mockClear(mapped);
 
         logger.info.mockClear();
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: false, currentFileVersion: 13, otaFileVersion: 13});
         MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
@@ -419,7 +425,7 @@ describe('OTA update', () => {
         expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
 
         logger.info.mockClear();
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce({available: true, currentFileVersion: 13, otaFileVersion: 15});
         MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
@@ -468,7 +474,7 @@ describe('OTA update', () => {
     it('Set zigbee_ota_override_index_location', async () => {
         settings.set(['ota', 'zigbee_ota_override_index_location'], 'local.index.json');
         await resetExtension();
-        expect(spyUseIndexOverride).toHaveBeenCalledWith(data.mockDir + '/local.index.json');
+        expect(spyUseIndexOverride).toHaveBeenCalledWith(path.join(data.mockDir, 'local.index.json'));
         spyUseIndexOverride.mockClear();
         
         settings.set(['ota', 'zigbee_ota_override_index_location'], 'http://my.site/index.json');

test/publish.test.js

@@ -507,6 +507,18 @@ describe('Publish', () => {
         expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
     });
 
+    it('Should handle get with multiple cover endpoints', async () => {
+        const device = zigbeeHerdsman.devices.zigfred_plus;
+        const endpoint11 = device.getEndpoint(11);
+        const endpoint12 = device.getEndpoint(12);
+        await MQTT.events.message('zigbee2mqtt/zigfred_plus/get', stringify({state_l6: '', state_l7: ''}));
+        await flushPromises();
+        expect(endpoint11.read).toHaveBeenCalledTimes(1);
+        expect(endpoint11.read).toHaveBeenCalledWith('closuresWindowCovering', ['currentPositionLiftPercentage']);
+        expect(endpoint12.read).toHaveBeenCalledTimes(1);
+        expect(endpoint12.read).toHaveBeenCalledWith('closuresWindowCovering', ['currentPositionLiftPercentage']);
+    });
+
     it('Should log error when device has no such endpoint (via topic)', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint2 = device.getEndpoint(2);
@@ -919,11 +931,11 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 0}, {}]);
-        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 1}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should turn device off when brightness 0 is send with transition', async () => {
@@ -938,11 +950,11 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
-        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 1}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should allow to set color via hue and saturation', async () => {

test/receive.test.js

@@ -16,6 +16,8 @@ describe('Receive', () => {
         jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        await jest.runOnlyPendingTimers();
+        await flushPromises();
     });
 
     beforeEach(async () => {
@@ -84,7 +86,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20, state: 'OPEN'}), {retain: false, qos: 0}, expect.any(Function));
 
         // Inverted
         MQTT.publish.mockClear();
@@ -92,7 +94,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80, state: 'OPEN'}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should allow to disable the legacy integration', async () => {
@@ -242,51 +244,15 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
-    it('Should handle a zigbee message with voltage 3010', async () => {
+    it('Should handle a zigbee message with voltage 2990', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 3010}}
+        const data = {'65281': {'1': 2990}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2850', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2850}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2650', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2650}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2000', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2000}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 93, voltage: 2990});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -298,7 +264,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'temperature': 19});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'device_temperature': 19, 'power_outage_count': 34});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -455,7 +421,7 @@ describe('Receive', () => {
         const data = {instantaneousDemand:496,currentSummDelivered:[0,6648]}
 
         const SP600_NEW = zigbeeHerdsman.devices.SP600_NEW;
-        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_NEW, endpoint: SP600_NEW.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_NEW, endpoint: SP600_NEW.getEndpoint(1), type: 'attributeReport', linkquality: 10, meta: {zclTransactionSequenceNumber: 1}});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
@@ -463,7 +429,7 @@ describe('Receive', () => {
 
         MQTT.publish.mockClear();
         const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
-        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_OLD, endpoint: SP600_OLD.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_OLD, endpoint: SP600_OLD.getEndpoint(1), type: 'attributeReport', linkquality: 10, meta: {zclTransactionSequenceNumber: 2}});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');

test/settings.test.js

@@ -39,6 +39,7 @@ describe('Settings', () => {
 
     beforeEach(() => {
         remove(configurationFile);
+        remove(secretFile);
         remove(devicesFile);
         remove(groupsFile);
         clearEnvironmentVariables();
@@ -66,20 +67,42 @@ describe('Settings', () => {
     it('Should apply environment variables', () => {
         process.env['ZIGBEE2MQTT_CONFIG_SERIAL_DISABLE_LED'] = 'true';
         process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_SOFT_RESET_TIMEOUT'] = 1;
-        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_OUTPUT'] = 'csvtest';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_OUTPUT'] = 'attribute_and_json';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_LOG_OUTPUT'] = '["console"]';
         process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
         process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
+        process.env['ZIGBEE2MQTT_CONFIG_MQTT_SERVER'] = 'testserver';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_NETWORK_KEY'] = 'GENERATE';
+        process.env['ZIGBEE2MQTT_CONFIG_DEVICES'] = 'devices.yaml';
+
+        const contentDevices = {
+            '0x00158d00018255df': {
+                friendly_name: '0x00158d00018255df',
+                retain: false,
+            },
+        };
 
         write(configurationFile, {});
+        write(devicesFile, contentDevices);
+        expect(settings.validate()).toStrictEqual([]);
+
         const s = settings.get();
         const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
-        expected.devices = {};
+        expected.devices = {
+            '0x00158d00018255df': {
+                friendly_name: '0x00158d00018255df',
+                retain: false,
+            },
+        };
         expected.groups = {};
         expected.serial.disable_led = true;
         expected.advanced.soft_reset_timeout = 1;
-        expected.advanced.output = 'csvtest';
+        expected.advanced.log_output = ["console"];
+        expected.advanced.output = 'attribute_and_json';
         expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
         expected.mqtt.base_topic = 'testtopic';
+        expected.mqtt.server = 'testserver';
+        expected.advanced.network_key = 'GENERATE';
 
         expect(s).toStrictEqual(expected);
     });
@@ -171,6 +194,49 @@ describe('Settings', () => {
         expect(read(secretFile)).toStrictEqual({...contentSecret, username: 'test123', network_key: [1,2,3,4]});
     });
 
+    it('Should read ALL secrets form a separate file', () => {
+        const contentConfiguration = {
+            mqtt: {
+                server: '!secret server',
+                user: '!secret username',
+                password: '!secret.yaml password',
+            },
+            advanced: {
+                network_key: '!secret network_key',
+            }
+        };
+
+        const contentSecret = {
+            server: 'my.mqtt.server',
+            username: 'mysecretusername',
+            password: 'mysecretpassword',
+            network_key: [1,2,3],
+        };
+
+        write(secretFile, contentSecret, false);
+        write(configurationFile, contentConfiguration);
+
+        const expected = {
+            base_topic: 'zigbee2mqtt',
+            include_device_information: false,
+            force_disable_retain: false,
+            password: "mysecretpassword",
+            server: "my.mqtt.server",
+            user: "mysecretusername",
+        };
+
+        expect(settings.get().mqtt).toStrictEqual(expected);
+        expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
+
+        settings.testing.write();
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual(contentSecret);
+
+        settings.set(['mqtt', 'server'], 'not.secret.server');
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual({...contentSecret, server: 'not.secret.server'});
+    });
+
     it('Should read devices form a separate file', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',
@@ -747,15 +813,15 @@ describe('Settings', () => {
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
-    it('Configuration friendly name cannot contain null char', async () => {
+    it('Configuration friendly name cannot contain control char', async () => {
         write(configurationFile, {
             ...minimalConfig,
-            devices: {'0x0017880104e45519': {friendly_name: 'blaa/blaa' + String.fromCharCode(0), retain: false}},
+            devices: {'0x0017880104e45519': {friendly_name: 'blaa/blaa\u009f', retain: false}},
         });
 
         settings.reRead();
 
-        const error = `friendly_name is not allowed to contain null char`;
+        const error = `friendly_name is not allowed to contain control char`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 

test/stub/data.js

@@ -21,7 +21,8 @@ function writeDefaultConfiguration() {
         devices: {
             "0x000b57fffec6a5b2": {
                 retain: true,
-                friendly_name: "bulb"
+                friendly_name: "bulb",
+                description: "this is my bulb",
             },
             "0x0017880104e45517": {
                 retain: true,
@@ -152,6 +153,23 @@ function writeDefaultConfiguration() {
             '0x0017880104e43559': {
                 friendly_name: 'U202DST600ZB'
             },
+            '0xf4ce368a38be56a1': {
+                retain: false,
+                friendly_name: 'zigfred_plus',
+                front_surface_enabled: 'true',
+                dimmer_1_enabled: 'true',
+                dimmer_1_dimming_enabled: 'true',
+                dimmer_2_enabled: 'true',
+                dimmer_2_dimming_enabled: 'true',
+                dimmer_3_enabled: 'true',
+                dimmer_3_dimming_enabled: 'true',
+                dimmer_4_enabled: 'true',
+                dimmer_4_dimming_enabled: 'true',
+                cover_1_enabled: 'true',
+                cover_1_tilt_enabled: 'true',
+                cover_2_enabled: 'true',
+                cover_2_tilt_enabled: 'true',
+            },
             '0x0017880104e44559': {
                 friendly_name: '3157100_thermostat',
             },
@@ -242,6 +260,9 @@ const defaultState = {
     "0x0017880104e45517": {
         "brightness": 255
     },
+    "1": {
+        'state': 'ON',
+    }
 }
 
 function getDefaultState() {

test/stub/logger.js

@@ -12,6 +12,7 @@ const callTransports = (level, message) => {
 }
 
 const mock = {
+    init: jest.fn(),
     info: jest.fn().mockImplementation((msg) => callTransports('info', msg)),
     warn: jest.fn().mockImplementation((msg) => callTransports('warn', msg)),
     error: jest.fn().mockImplementation((msg) => callTransports('error', msg)),

test/stub/zigbeeHerdsman.js

@@ -27,7 +27,11 @@ const clusters = {
     'lightingColorCtrl': 768,
     'closuresWindowCovering': 258,
     'hvacThermostat': 513,
+    'msIlluminanceMeasurement': 1024,
     'msTemperatureMeasurement': 1026,
+    'msRelativeHumidity': 1029,
+    'msSoilMoisture': 1032,
+    'msCO2': 1037
 }
 
 class Endpoint {
@@ -133,6 +137,7 @@ const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,41
 const ZNCZ02LM = new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug");
 const GLEDOPTO_2ID = new Device('Router', '0x0017880104e45724', 6540,4151, [new Endpoint(11, [0,3,4,5,6,8,768], [], '0x0017880104e45724', [], {}, [], 49246, 528), new Endpoint(12, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 260, 258), new Endpoint(13, [4096], [4096], '0x0017880104e45724', [], {}, [], 49246, 57694), new Endpoint(15, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 49246, 256)], true, "Mains (single phase)", 'GL-C-007', false, 'GLEDOPTO');
 const QBKG03LM = new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2');
+const zigfred_plus = new Device('Router', '0xf4ce368a38be56a1', 6589, 0x129C, [new Endpoint(5, [0, 3, 4, 5, 6, 8, 0x0300, 0xFC42], [0xFC42], '0xf4ce368a38be56a1'), new Endpoint(7, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(8, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(9, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(10, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(11, [0, 3, 4, 5, 0x0102], [], '0xf4ce368a38be56a1'), new Endpoint(12, [0, 3, 4, 5, 0x0102], [], '0xf4ce368a38be56a1')], true, "Mains (single phase)", 'zigfred plus', false, 'Siglis');
 
 const groups = {
     'group_1': new Group(1, []),
@@ -188,7 +193,8 @@ const devices = {
     'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
     'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
     'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
-    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
+    'zigfred_plus': zigfred_plus,
+    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100', false, 'Centralite'),
     'J1': new Device('Router', '0x0017880104a44559', 6543,4151, [new Endpoint(1, [], [], '0x0017880104a44559')], true, "Mains (single phase)", 'J1 (5502)'),
     'TS0601_thermostat': TS0601_thermostat,
     'external_converter_device': new Device('EndDevice', '0x0017880104e45511', 1114, 'external', [new Endpoint(1, [], [], '0x0017880104e45511')], false, null, 'external_converter_device' ),
@@ -197,7 +203,7 @@ const devices = {
     'temperature_sensor': new Device('EndDevice', '0x0017880104e45561', 6544,4151, [new Endpoint(1, [0,3,4,1026], [])], true, "Battery", "temperature.sensor"),
     'heating_actuator': new Device('Router', '0x0017880104e45562', 6545,4151, [new Endpoint(1, [0,3,4,513], [1026])], true, "Mains (single phase)", "heating.actuator"),
     'bj_scene_switch': new Device('EndDevice', '0xd85def11a1002caa', 50117, 4398, [new Endpoint(10, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa', [{target: bulb_color_2.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 6, name: 'genOnOff'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 768, name: 'lightingColorCtrl'}},]), new Endpoint(11, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa')], true, 'Battery', 'RB01', false, 'Busch-Jaeger', '20161222', '1.2.0'),
-
+    'GW003-AS-IN-TE-FC': new Device('Router', '0x0017548104a44669', 6545,4699, [new Endpoint(1, [3], [0,3,513,514], '0x0017548104a44669')], true, "Mains (single phase)", 'Adapter Zigbee FUJITSU'),
 }
 
 const mock = {
@@ -207,8 +213,10 @@ const mock = {
     touchlinkScan: jest.fn(),
     touchlinkIdentify: jest.fn(),
     start: jest.fn(),
+    backup: jest.fn(),
     isStopping: jest.fn(),
     permitJoin: jest.fn(),
+    addInstallCode: jest.fn(),
     getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
     getNetworkParameters: jest.fn().mockReturnValue({panID: 0x162a, extendedPanID: [0, 11, 22], channel: 15}),
     on: (type, handler) => {

update.sh

@@ -18,7 +18,6 @@ echo "Creating backup of configuration..."
 cp -R data data-backup
 
 echo "Updating..."
-git checkout HEAD -- npm-shrinkwrap.json
 git pull
 
 echo "Installing dependencies..."