1.28.4

Undev
Update zhc to 14.0.681-hotfix.0
2026-08-29 15:18:41 +00:00 · 2022-12-01 19:58:42 +01:00 · 2022-12-01 19:58:39 +01:00 · 2022-12-01 19:56:37 +01:00 · 2022-12-01 17:38:11 +01:00 · 2022-12-01 17:35:02 +01:00
85 changed files with 19733 additions and 24892 deletions
@@ -15,6 +15,7 @@ module.exports = {
        'indent': ['error', 4],
        'max-len': ['error', {'code': 120}],
        'no-prototype-builtins': 'off',
+        'linebreak-style': ['error', (process.platform === 'win32' ? 'windows' : 'unix')], // https://stackoverflow.com/q/39114446/2771889
    },
    'plugins': [
        'jest',
@@ -1,46 +0,0 @@
-<!--
-Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
-
-Rules (don't ignore these, your issue will be closed without further notice):
- English only
- Make sure you are on the latest Zigbee2MQTT version
- Provide a clear description of the problem
- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
-
-Did you read the FAQ?
- https://www.zigbee2mqtt.io/information/FAQ.html
-
-Zigbee2MQTT fails to start?
- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
-
-Having issues when using a CC2531?
- Make sure the CC2531 is connected through a USB extension cable
- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
-
-Unsupported device?
- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
-
-Device does not pair or interview fails?
- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
-
-Bug report?
- If applicable, provide steps how to reproduce the problem.
- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
-
-->
-
-<!--Start your bug report below this line-->
-
-# Bug Report
-## What happened
-
-## What did you expect to happen
-
-## How to reproduce it (minimal and precise)
-
-## Debug Info
-Zigbee2MQTT version:
-Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
-Adapter firmware version:
@@ -10,7 +10,7 @@ contact_links:
    url: https://github.com/zigbee2mqtt/hassio-zigbee2mqtt/issues
    about: Issues related to the Home Assistant addon.
  - name: FAQ
-    url: https://www.zigbee2mqtt.io/information/FAQ.html
+    url: https://www.zigbee2mqtt.io/guide/faq
    about: Frequently asked questions.
  - name: Support Chat
    url: https://discord.gg/NyseBeK
@@ -1,20 +0,0 @@
---
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: feature request
-assignees: ''
-
---
-
-## Is your feature request related to a problem? Please describe
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-## Describe the solution you'd like
-A clear and concise description of what you want to happen.
-
-## Describe alternatives you've considered
-A clear and concise description of any alternative solutions or features you've considered.
-
-## Additional context
-Add any other context or screenshots about the feature request here.
@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+title: "[Feature request]: "
+labels: [feature request]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        **IMPORTANT:** Before submitting:
+        - Is your feature request related to the fronted? Then click [here](https://github.com/nurikk/zigbee2mqtt-frontend/issues/new?assignees=&labels=&template=feature_request.md&title=)
+  - type: textarea
+    id: textarea1
+    attributes:
+      label: Is your feature request related to a problem? Please describe
+      placeholder: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: textarea2
+    attributes:
+      label: Describe the solution you'd like
+      placeholder: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: textarea3
+    attributes:
+      label: Describe alternatives you've considered
+      placeholder: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: textarea4
+    attributes:
+      label: Additional context
+      placeholder: Add any other context or screenshots about the feature request here.
+    validations:
+      required: true
@@ -1,21 +0,0 @@
---
-name: New device support request
-about: Request support for a new device
-title: ''
-labels: new device support
-assignees: ''
-
---
-
-<!--
-VERY IMPORTANT: Before submitting a request, first (try to) follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
-
-Dumping just the info about this device without actually attemping adding support yourself will likely lead into this issue being ignored.
-Clearly indicate what you tried and where you got stuck.
-->
-
-## Information about the device + link
-..
-
-## data/database.db entry of the device
-..
@@ -0,0 +1,55 @@
+name: New device support request
+description: Request support for a new device
+title: "[New device support]: "
+labels: [new device support]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        **IMPORTANT:** Before submitting:
+        - Make sure this device is not already supported in the [dev branch](https://www.zigbee2mqtt.io/advanced/more/switch-to-dev-branch.html)
+        - Make sure there is no existing issue or PR for this device already, search for your device here: https://github.com/Koenkk/zigbee2mqtt/issues
+        - Follow this [guide](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html)
+        - If you are using the Home Assistant addon and are still on 1.18.1, check the first point of [this](https://github.com/Koenkk/zigbee2mqtt/releases/tag/1.19.0)
+  - type: input
+    id: link
+    attributes:
+      label: Link
+      description: Link of this device (product page)
+      placeholder: https://www.linktomydevice.org
+    validations:
+      required: true
+  - type: input
+    id: database
+    attributes:
+      label: Database entry
+      description: Entry of this device in `data/database.db` after pairing it
+      placeholder: '{"id":53,"type":"Router","ieeeAddr":"0x10458d00024284f69","nwkAddr":10148,"manufId":4151,"manufName":"LUMI","powerSource":"DC Source","modelId":"lumi.relay.c2acn01","epList":[1,2],"endpoints":{"1":{"profId":260,"epId":1,"devId":257,"inClusterList":[0,3,4,5,1,2,10,6,16,2820,12],"outClusterList":[25,10],"clusters":{"genBasic":{"attributes":{"modelId":"lumi.relay.c2acn01","appVersion":1,"manufacturerName":"LUMI","powerSource":4,"zclVersion":0,"stackVersion":2,"hwVersion":18,"dateCode":"8-6-2020"}},"genAnalogInput":{"attributes":{"presentValue":129.04425048828125}},"genOnOff":{"attributes":{"61440":117440715,"onOff":1}}},"binds":[],"configuredReportings":[],"meta":{}},"2":{"profId":260,"epId":2,"devId":257,"inClusterList":[6,16,4,5],"outClusterList":[],"clusters":{"genOnOff":{"attributes":{"61440":237478966,"onOff":0}}},"binds":[],"configuredReportings":[],"meta":{}}},"appVersion":1,"stackVersion":2,"hwVersion":18,"dateCode":"8-6-2020","zclVersion":0,"interviewCompleted":true,"meta":{},"lastSeen":1640285631405}'
+    validations:
+      required: true
+  - type: textarea
+    id: notes
+    attributes:
+      label: Comments
+      placeholder: I tried to follow the supporting new device page but got stuck at...
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: External converter
+      description: In case you tried to create an external converter, put it below
+      render: shell
+  - type: markdown
+    attributes:
+      value: |
+        # Lights
+        In case this device is a light, fill in the fields below:
+  - type: input
+    attributes:
+      label: Supported color modes
+      placeholder: color temperature, color (rgb)
+  - type: input
+    attributes:
+      label: Color temperature range
+      description: In case this light supports color temperature, provide the color temperature range. [How to do this](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html#_3-1-retrieving-color-temperature-range-only-required-for-lights-which-support-color-temperature).
+      placeholder: '{"colorTempPhysicalMin":153,"colorTempPhysicalMax":454}'
@@ -1,54 +0,0 @@
---
-name: Problem report
-about: Create a report to help us improve
-title: ''
-labels: problem
-assignees: ''
-
---
-
-<!--
-Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
-
-Rules (don't ignore these, your issue will be closed without further notice):
- English only
- Make sure you are on the latest Zigbee2MQTT version
- Provide a clear description of the problem
- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
-
-Did you read the FAQ?
- https://www.zigbee2mqtt.io/information/FAQ.html
-
-Zigbee2MQTT fails to start?
- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
-
-Having issues when using a CC2531?
- Make sure the CC2531 is connected through a USB extension cable
- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
-
-Unsupported device?
- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
-
-Device does not pair or interview fails?
- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
-
-Bug report?
- If applicable, provide steps how to reproduce the problem.
- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
-
-->
-
-<!--Start your bug report below this line-->
-
-## What happened
-
-## What did you expect to happen
-
-## How to reproduce it (minimal and precise)
-
-## Debug info
-Zigbee2MQTT version:
-Adapter hardware: CC2531, CC2530, CC26X2R1, CC1352P-2, Conbee II
-Adapter firmware version:
@@ -0,0 +1,67 @@
+name: Problem report
+description: Create a report to help us improve
+labels: [problem]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        **IMPORTANT:** Before submitting:
+        - You read the [FAQ](https://www.zigbee2mqtt.io/guide/faq/)
+        - Zigbee2MQTT fails to start? Read [this](https://www.zigbee2mqtt.io/guide/installation/20_zigbee2mqtt-fails-to-start.html)
+        - Make sure the bug also occurs in the [dev branch](https://www.zigbee2mqtt.io/advanced/more/switch-to-dev-branch.html)
+        - Make sure you are using the [latest firmware](https://www.zigbee2mqtt.io/guide/adapters/#recommended) on your adapter
+        - The issue has not been [reported already](https://github.com/Koenkk/zigbee2mqtt/issues)
+        - Is your issue related to the fronted? Then click [here](https://github.com/nurikk/zigbee2mqtt-frontend/issues/new?assignees=&labels=bug%2Ctriage&template=bug_report.yaml&title=%5BBug%5D%3A+)
+        - If you are using the Home Assistant addon and are still on 1.18.1, check the first point of [this](https://github.com/Koenkk/zigbee2mqtt/releases/tag/1.19.0)
+  - type: textarea
+    id: what_happend
+    attributes:
+      label: What happened?
+    validations:
+      required: true
+  - type: textarea
+    id: expect_to_happen
+    attributes:
+      label: What did you expect to happen?
+      placeholder: I expected that ...
+    validations:
+      required: false
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: How to reproduce it (minimal and precise)
+      placeholder: First do this, than this..
+    validations:
+      required: false
+  - type: input
+    id: z2m_version
+    attributes:
+      label: Zigbee2MQTT version
+      description: Can be found in the frontend -> settings -> about -> Zigbee2MQTT version. Are you running Zigbee2MQTT 1.18.1? Then read [this](https://github.com/Koenkk/zigbee2mqtt/releases/tag/1.19.0).
+      placeholder: '1.22.1'
+    validations:
+      required: true
+  - type: input
+    id: adapter_fwversion
+    attributes:
+      label: Adapter firmware version
+      description: Can be found in the frontend -> settings -> about -> coordinator revision
+      placeholder: '20211210'
+    validations:
+      required: true
+  - type: input
+    id: adapter
+    attributes:
+      label: Adapter
+      description: The adapter you are using
+      placeholder: Electrolama zig-a-zig-ah! (zzh!), Slaesh's CC2652RB stick, ...
+    validations:
+      required: true
+  - type: textarea
+    id: log
+    attributes:
+      label: Debug log
+      description: After enabling [debug logging](https://www.zigbee2mqtt.io/guide/usage/debug.html) the log can be found under `data/log`. Attach the file below
+      placeholder: Click here and drag the file into it or click on "Attach files by.." below
+    validations:
+      required: false
@@ -1 +0,0 @@
-preferredBranch: dev
@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    target-branch: dev
+
+  - package-ecosystem: "docker"
+    directory: "/docker"
+    schedule:
+      interval: "daily"
+    target-branch: dev
@@ -2,27 +2,20 @@ name: CI

 on: [push, pull_request]

+permissions:
+  contents: read
+
 jobs:
  ci:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
      with:
-        node-version: 14
+        node-version: 16
        registry-url: https://registry.npmjs.org/
-    - name: Get NodeJS Version
-      id: nodeversion
-      run: |
-        echo "::set-output name=version::$(node --version)"
-    - name: Restore node_modules cache
-      id: cache-node-modules
-      uses: actions/cache@v2
-      with:
-        path: node_modules
-        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+        cache: 'npm'
    - name: Install dependencies
-      if: steps.cache-node-modules.outputs.cache-hit != 'true'
      run: npm ci
    - name: Build
      run: npm run build
@@ -38,13 +31,13 @@ jobs:
      run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u koenkk --password-stdin
    - name: Docker setup - QEMU
      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
      with:
        platforms: all
    - name: Docker setup - Buildx
      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
      id: buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
      with:
        version: latest
    - name: Docker build dev
@@ -76,9 +69,33 @@ jobs:
    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
      run: |
-        curl -s \
-          --request POST \
-          --header "Content-Type: application/json" \
-          --data '{"definition": {"id": 9}}' \
-          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
-          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"
+        curl \
+          -X POST \
+          -H "Authorization: token ${{ secrets.CR_PAT }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          https://api.github.com/repos/zigbee2mqtt/hassio-zigbee2mqtt/actions/workflows/ci.yml/dispatches \
+          -d '{"ref":"master","inputs":{}}'
+
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        node: [14, 16, 18, 19]
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: https://registry.npmjs.org/
+          cache: 'npm'
+      - name: Install dependencies
+        # --ignore-scripts prevents the serialport build which often fails on Windows
+        run: npm ci --ignore-scripts
+      - name: Lint
+        run: npm run eslint
+      - name: Build
+        run: npm run build
+      - name: Test
+        run: npm run test-with-coverage
@@ -0,0 +1,34 @@
+name: "Code Scanning - Action"
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  CodeQL-Build:
+    # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
+    runs-on: ubuntu-latest
+
+    permissions:
+      # required for all workflows
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+
+      # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below).
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      #- run: |
+      #     make bootstrap
+      #     make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
@@ -0,0 +1,14 @@
+name: 'Dependency review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency review'
+        uses: actions/dependency-review-action@v3
@@ -0,0 +1,34 @@
+on:
+  workflow_dispatch:
+
+name: GHCR cleanup
+
+permissions: {}
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Delete untagged images
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.CR_PAT }}
+          script: |
+            const response = await github.request("GET /${{ env.OWNER }}/packages/container/${{ env.PACKAGE_NAME }}/versions",
+              { per_page: ${{ env.PER_PAGE }}
+            });
+            for(version of response.data) {
+                if (version.metadata.container.tags.length == 0) {
+                    try {
+                        console.log("delete " + version.id)
+                        const deleteResponse = await github.request("DELETE /${{ env.OWNER }}/packages/container/${{ env.PACKAGE_NAME }}/versions/" + version.id, { });
+                        console.log("status " + deleteResponse.status)
+                    } catch (e) {
+                        console.log("failed")
+                    }
+                }
+            }
+        env:
+          OWNER: user
+          PACKAGE_NAME: zigbee2mqtt
+          PER_PAGE: 2000
@@ -4,11 +4,17 @@ on:
  - cron: "0 0 * * *"
  workflow_dispatch:

+permissions:
+  contents: read
+
 jobs:
  stale:
+    permissions:
+      issues: write  # for actions/stale to close stale issues
+      pull-requests: write  # for actions/stale to close stale PRs
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v6
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
@@ -16,4 +22,4 @@ jobs:
        days-before-stale: 30
        days-before-close: 7
        operations-per-run: 500
-        exempt-issue-labels: 'breaking change'
+        exempt-issue-labels: dont-stale
@@ -4,21 +4,26 @@ on:
  schedule:
  - cron: "0 0 * * 0"

+permissions:
+  contents: read
+
 jobs:
  update_deps:
+    permissions:
+      contents: write  # for peter-evans/create-pull-request to create branch
+      pull-requests: write  # for peter-evans/create-pull-request to create a PR
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        ref: dev
        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
      with:
-        node-version: 14
-    - run: rm -rf npm-shrinkwrap.json
-    - run: npx npm-check-updates -u
-    - run: npm install && npm shrinkwrap --dev
-    - uses: peter-evans/create-pull-request@v3
+        node-version: 16
+    - run: npx npm-check-updates -u -x connect-gzip-static # connect-gzip-static only supports node 16
+    - run: npm install
+    - uses: peter-evans/create-pull-request@v4
      with:
        commit-message: Update dependencies
        branch: "update_deps"
@@ -3,20 +3,33 @@ on:
  repository_dispatch:
    types: update_frontend

+permissions: {}
 jobs:
  update_frontend:
+    permissions:
+      contents: write  #  to create branch (peter-evans/create-pull-request)
+      pull-requests: write  #  to create a PR (peter-evans/create-pull-request)
+
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        ref: dev
        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
      with:
-        node-version: 14
+        node-version: 16
    - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v3
+    - uses: peter-evans/create-pull-request@v4
+      id: cpr
      with:
        commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
        branch: "update_frontend"
        title: Update zigbee-frontend to ${{ github.event.client_payload.version }}
+    - run: sleep 5 # Otherwise pull request may not exist yet causing automerge to fail
+    - uses: peter-evans/enable-pull-request-automerge@v2
+      if: steps.cpr.outputs.pull-request-operation == 'created'
+      with:
+        token: ${{ secrets.CR_PAT }}
+        pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
+        merge-method: squash
@@ -3,20 +3,33 @@ on:
  repository_dispatch:
    types: update_zh

+permissions: {}
 jobs:
  update_zh:
+    permissions:
+      contents: write  #  to create branch (peter-evans/create-pull-request)
+      pull-requests: write  #  to create a PR (peter-evans/create-pull-request)
+
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        ref: dev
        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
      with:
-        node-version: 14
+        node-version: 16
    - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v3
+    - uses: peter-evans/create-pull-request@v4
+      id: cpr
      with:
        commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
        branch: "update_zh"
        title: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
+    - run: sleep 5 # Otherwise pull request may not exist yet causing automerge to fail
+    - uses: peter-evans/enable-pull-request-automerge@v2
+      if: steps.cpr.outputs.pull-request-operation == 'created'
+      with:
+        token: ${{ secrets.CR_PAT }}
+        pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
+        merge-method: squash
@@ -3,20 +3,33 @@ on:
  repository_dispatch:
    types: update_zhc

+permissions: {}
 jobs:
  update_zhc:
+    permissions:
+      contents: write  #  to create branch (peter-evans/create-pull-request)
+      pull-requests: write  #  to create a PR (peter-evans/create-pull-request)
+
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
      with:
        ref: dev
        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-    - uses: actions/setup-node@v2-beta
+    - uses: actions/setup-node@v3
      with:
-        node-version: 14
+        node-version: 16
    - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v3
+    - uses: peter-evans/create-pull-request@v4
+      id: cpr
      with:
        commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
        branch: "update_zhc"
        title: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
+    - run: sleep 5 # Otherwise pull request may not exist yet causing automerge to fail
+    - uses: peter-evans/enable-pull-request-automerge@v2
+      if: steps.cpr.outputs.pull-request-operation == 'created'
+      with:
+        token: ${{ secrets.CR_PAT }}
+        pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
+        merge-method: squash
@@ -3,6 +3,6 @@
 Everybody is invited and welcome to contribute to Zigbee2MQTT. Zigbee2MQTT is writen in JavaScript and is based upon [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) and [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters). Zigbee-herdsman-converters contains all device definition, zigbee-herdsman is responsible for handling all communication with the adapter.

 - Pull requests are always created against the [**dev**](https://github.com/Koenkk/zigbee2mqtt/tree/dev) branch.
- Easiest way to start developing Zigbee2MQTT is by setting up a development environment (aka bare-metal installation). You can follow this [guide](https://www.zigbee2mqtt.io/getting_started/running_zigbee2mqtt.html#running-zigbee2mqtt) to do this.
- You can run the tests locally by executing `npm test`. Zigbee2MQTT enforces 100% code coverage, in case you add new code check if you code is covered by running `npm run test-with-coverage`. The coverage report can be found under `coverage/lcov-report/index.html`. Linting is also enforced and can be run with `npm run eslint`.
- When you want to add support for a new devices no changes to Zigbee2MQTT have to be made, only to zigbee-herdsman-converters. You can find a guide for it [here](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).
+- Easiest way to start developing Zigbee2MQTT is by setting up a development environment (aka bare-metal installation). You can follow this [guide](https://www.zigbee2mqtt.io/guide/installation/01_linux.html) to do this.
+- You can run the tests locally by executing `npm test`. Zigbee2MQTT enforces 100% code coverage, in case you add new code check if your code is covered by running `npm run test-with-coverage`. The coverage report can be found under `coverage/lcov-report/index.html`. Linting is also enforced and can be run with `npm run eslint`.
+- When you want to add support for a new devices no changes to Zigbee2MQTT have to be made, only to zigbee-herdsman-converters. You can find a guide for it [here](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html).
@@ -35,26 +35,26 @@
    </div>
    <h1>Zigbee2MQTT  🌉 🐝</h1>
    <p>
-        Allows you to use your Zigbee devices <b>without</b> the vendors bridge or gateway.
+        Allows you to use your Zigbee devices <b>without</b> the vendor's bridge or gateway.
    </p>
    <p>
        It bridges events and allows you to control your Zigbee devices via MQTT. In this way you can integrate your Zigbee devices with whatever smart home infrastructure you are using.
    </p>
 </div>

-## [Getting started](https://www.zigbee2mqtt.io/#getting-started)
+## [Getting started](https://www.zigbee2mqtt.io/guide/getting-started)
 The [documentation](https://www.zigbee2mqtt.io/) provides you all the information needed to get up and running! Make sure you don't skip sections if this is your first visit, as there might be important details in there for you.

-If you aren't familiar with **Zigbee** terminology make sure you [read this](https://www.zigbee2mqtt.io/information/zigbee_network.html) to help you out.
+If you aren't familiar with **Zigbee** terminology make sure you [read this](https://www.zigbee2mqtt.io/advanced/zigbee/01_zigbee_network.html) to help you out.

-## [Integrations](https://www.zigbee2mqtt.io/#integration)
+## [Integrations](https://www.zigbee2mqtt.io/guide/usage/integrations.html)
 Zigbee2MQTT integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:

 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">

 ### [Home Assistant](https://www.home-assistant.io/)
 - [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/zigbee2mqtt/hassio-zigbee2mqtt)
- Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
+- Generic install: using instructions [here](https://www.zigbee2mqtt.io/guide/usage/integrations/home_assistant.html)

 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">

@@ -67,15 +67,15 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 ![Architecture](images/architecture.png)

 ### Internal Architecture
-Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter and makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/blob/master/docs/07-5123-08-Zigbee-Cluster-Library.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities. Zigbee2MQTT provides a [web-based interface](https://github.com/nurikk/zigbee2mqtt-frontend) that allows monitoring and configuration.

 ### Developing
 Zigbee2MQTT uses TypeScript (partially for now). Therefore after making changes to files in the `lib/` directory you need to recompile Zigbee2MQTT. This can be done by executing `npm run build`. For faster development instead of running `npm run build` you can run `npm run build-watch` in another terminal session, this will recompile as you change files.

 ## Supported devices
-See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
+See [Supported devices](https://www.zigbee2mqtt.io/supported-devices) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.

-If it's not listed in [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html), support can be added (fairly) easy, see [How to support new devices](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).
+If it's not listed in [Supported devices](https://www.zigbee2mqtt.io/supported-devices), support can be added (fairly) easily, see [How to support new devices](https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html).

 ## Support & help
 If you need assistance you can check [opened issues](https://github.com/Koenkk/zigbee2mqtt/issues). Feel free to help with Pull Requests when you were able to fix things or add new devices or just share the love on social media.
@@ -1,20 +1,15 @@
-FROM node:14-alpine3.12 as base
+FROM node:18-alpine3.16 as base

 WORKDIR /app
 RUN apk add --no-cache tzdata eudev tini

-COPY package.json ./
-
 # Dependencies and build
 FROM base as dependencies_and_build

-COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY package*.json tsconfig.json index.js ./
 COPY lib ./lib

 RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
-    npm ci --no-audit --no-optional --no-update-notifier && \
-    npm run build && \
-    rm -rf node_modules && \
    npm ci --production --no-audit --no-optional --no-update-notifier && \
    apk del .buildtools

@@ -22,8 +17,8 @@ RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers
 FROM base as release

 COPY --from=dependencies_and_build /app/node_modules ./node_modules
-COPY --from=dependencies_and_build /app/dist ./dist
-COPY LICENSE index.js data/configuration.yaml ./
+COPY dist ./dist
+COPY package.json LICENSE index.js data/configuration.yaml ./

 COPY docker/docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
@@ -33,5 +28,7 @@ RUN mkdir /app/data
 ARG COMMIT
 RUN echo "$COMMIT" > dist/.hash

+ENV NODE_ENV production
+
 ENTRYPOINT ["docker-entrypoint.sh"]
-CMD [ "/sbin/tini", "--", "node", "index.js"]
+CMD [ "/sbin/tini", "--", "node", "index.js"]
@@ -1,9 +1,7 @@
-require('core-js/features/object/from-entries');
-require('core-js/features/array/flat');
 const semver = require('semver');
 const engines = require('./package.json').engines;
-const indexJsRestart = 'indexjs.restart';
 const fs = require('fs');
+const os = require('os');
 const path = require('path');
 const {exec} = require('child_process');
 const rimraf = require('rimraf');
@@ -15,12 +13,12 @@ let stopping = false;
 const hashFile = path.join(__dirname, 'dist', '.hash');

 async function restart() {
-    await stop(indexJsRestart);
+    await stop(true);
    await start();
 }

-async function exit(code, reason) {
-    if (reason !== indexJsRestart) {
+async function exit(code, restart) {
+    if (!restart) {
        process.exit(code);
    }
 }
@@ -44,9 +42,21 @@ async function build(reason) {
    return new Promise((resolve, reject) => {
        process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
        rimraf.sync('dist');
-        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+        const env = {...process.env};
+        const _600mb = 629145600;
+        if (_600mb > os.totalmem() && !env.NODE_OPTIONS) {
+            // Prevent OOM on tsc compile for system with low memory
+            // https://github.com/Koenkk/zigbee2mqtt/issues/12034
+            env.NODE_OPTIONS = '--max_old_space_size=256';
+        }
+
+        exec('npm run build', {env, cwd: __dirname}, async (err, stdout, stderr) => {
            if (err) {
                process.stdout.write(', failed\n');
+                if (err.code === 134) {
+                    process.stderr.write(
+                        '\n\nBuild failed; ran out-of-memory, free some memory (RAM) and start again\n\n');
+                }
                reject(err);
            } else {
                process.stdout.write(', finished\n');
@@ -87,7 +97,7 @@ async function start() {
        for (const error of errors) {
            console.log(`- ${error}`);
        }
-        console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`); // eslint-disable-line
+        console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/guide/configuration`); // eslint-disable-line
        console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
        exit(1);
    }
@@ -97,21 +107,27 @@ async function start() {
    await controller.start();
 }

-async function stop(reason=null) {
-    await controller.stop(reason);
+async function stop(restart) {
+    await controller.stop(restart);
 }

 async function handleQuit() {
    if (!stopping && controller) {
        stopping = true;
-        await stop();
+        await stop(false);
    }
 }

-if (process.argv.length === 3 && process.argv[2] === 'writehash') {
-    writeHash();
+if (require.main === module || require.main.filename.endsWith(path.sep + 'cli.js')) {
+    if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+        writeHash();
+    } else {
+        process.on('SIGINT', handleQuit);
+        process.on('SIGTERM', handleQuit);
+        start();
+    }
 } else {
    process.on('SIGINT', handleQuit);
    process.on('SIGTERM', handleQuit);
-    start();
+    module.exports = {start};
 }
@@ -37,19 +37,20 @@ const AllExtensions = [
 ];

 type ExtensionArgs = [Zigbee, MQTT, State, PublishEntityState, EventBus,
-    (enable: boolean, name: string) => Promise<void>, () => void, (extension: Extension) => void];
+    (enable: boolean, name: string) => Promise<void>, () => void, (extension: Extension) => Promise<void>];

-class Controller {
+export class Controller {
    private eventBus: EventBus;
    private zigbee: Zigbee;
    private state: State;
    private mqtt: MQTT;
    private restartCallback: () => void;
-    private exitCallback: (code: number) => void;
+    private exitCallback: (code: number, restart: boolean) => void;
    private extensions: Extension[];
    private extensionArgs: ExtensionArgs;

-    constructor(restartCallback: () => void, exitCallback: (code: number) => void) {
+    constructor(restartCallback: () => void, exitCallback: (code: number, restart: boolean) => void) {
+        logger.init();
        this.eventBus = new EventBus( /* istanbul ignore next */ (error) => {
            logger.error(`Error: ${error.message}`);
            logger.debug(error.stack);
@@ -101,16 +102,17 @@ class Controller {
            this.eventBus.onAdapterDisconnected(this, this.onZigbeeAdapterDisconnected);
        } catch (error) {
            logger.error('Failed to start zigbee');
-            logger.error('Check https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start for possible solutions'); /* eslint-disable-line max-len */
+            logger.error('Check https://www.zigbee2mqtt.io/guide/installation/20_zigbee2mqtt-fails-to-start.html for possible solutions'); /* eslint-disable-line max-len */
            logger.error('Exiting...');
            logger.error(error.stack);
-            this.exitCallback(1);
+            await this.exit(1);
        }

        // Disable some legacy options on new network creation
        if (startResult === 'reset') {
            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['advanced', 'legacy_availability_payload'], false);
            settings.set(['device_options', 'legacy'], false);
            this.enableDisableExtension(false, 'BridgeLegacy');
        }
@@ -142,10 +144,9 @@ class Controller {
        try {
            await this.mqtt.connect();
        } catch (error) {
-            logger.error(`MQTT failed to connect: ${error.message}`);
-            logger.error('Exiting...');
+            logger.error(`MQTT failed to connect, exiting...`);
            await this.zigbee.stop();
-            this.exitCallback(1);
+            await this.exit(1);
        }

        // Call extensions
@@ -153,17 +154,17 @@ class Controller {

        // Send all cached states.
        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
-            for (const device of devices) {
-                if (this.state.exists(device)) {
-                    this.publishEntityState(device, this.state.get(device));
+            for (const entity of [...devices, ...this.zigbee.groups()]) {
+                if (this.state.exists(entity)) {
+                    this.publishEntityState(entity, this.state.get(entity), 'publishCached');
                }
            }
        }

-        if (settings.get().advanced.last_seen && settings.get().advanced.last_seen !== 'disable') {
-            this.eventBus.onLastSeenChanged(this, (data) =>
-                this.publishEntityState(data.device, {}, 'lastSeenChanged'));
-        }
+        this.eventBus.onLastSeenChanged(this,
+            (data) => utils.publishLastSeen(data, settings.get(), false, this.publishEntityState));
+
+        logger.info(`Zigbee2MQTT started!`);
    }

    @bind async enableDisableExtension(enable: boolean, name: string): Promise<void> {
@@ -187,7 +188,7 @@ class Controller {
        await this.callExtensions('start', [extension]);
    }

-    async stop(): Promise<void> {
+    async stop(restart = false): Promise<void> {
        // Call extensions
        await this.callExtensions('stop', this.extensions);
        this.eventBus.removeListeners(this);
@@ -199,13 +200,18 @@ class Controller {
        try {
            await this.zigbee.stop();
            logger.info('Stopped Zigbee2MQTT');
-            this.exitCallback(0);
+            await this.exit(0, restart);
        } catch (error) {
            logger.error('Failed to stop Zigbee2MQTT');
-            this.exitCallback(1);
+            await this.exit(1, restart);
        }
    }

+    async exit(code: number, restart = false): Promise<void> {
+        await logger.end();
+        this.exitCallback(code, restart);
+    }
+
    @bind async onZigbeeAdapterDisconnected(): Promise<void> {
        logger.error('Adapter disconnected, stopping');
        await this.stop();
@@ -224,11 +230,11 @@ class Controller {
        }

        const options: MQTTOptions = {
-            retain: utils.getObjectProperty(entity.settings, 'retain', false) as boolean,
-            qos: utils.getObjectProperty(entity.settings, 'qos', 0) as 0 | 1 | 2,
+            retain: utils.getObjectProperty(entity.options, 'retain', false) as boolean,
+            qos: utils.getObjectProperty(entity.options, 'qos', 0) as 0 | 1 | 2,
        };

-        const retention = utils.getObjectProperty(entity.settings, 'retention', false);
+        const retention = utils.getObjectProperty(entity.options, 'retention', false);
        if (retention !== false) {
            options.properties = {messageExpiryInterval: retention as number};
        }
@@ -256,17 +262,15 @@ class Controller {
            message.linkquality = entity.zh.linkquality;
        }

-        // filter mqtt message attributes
-        if (entity.settings.filtered_attributes) {
-            entity.settings.filtered_attributes.forEach((a) => delete message[a]);
-        }
-
        for (const extension of this.extensions) {
            extension.adjustMessageBeforePublish?.(entity, message);
        }

+        // Filter mqtt message attributes
+        utils.filterProperties(entity.options.filtered_attributes, message);
+
        if (Object.entries(message).length) {
-            const output = settings.get().experimental.output;
+            const output = settings.get().advanced.output;
            if (output === 'attribute_and_json' || output === 'json') {
                await this.mqtt.publish(entity.name, stringify(message), options);
            }
@@ -276,7 +280,7 @@ class Controller {
            }
        }

-        this.eventBus.emitPublishEntityState({entity, message, stateChangeReason});
+        this.eventBus.emitPublishEntityState({entity, message, stateChangeReason, payload});
    }

    async iteratePayloadAttributeOutput(topicRoot: string, payload: KeyValue, options: MQTTOptions): Promise<void> {
@@ -27,10 +27,17 @@ export default class EventBus {
        this.on('permitJoinChanged', callback, key);
    }

-    public emitDeviceRenamed(data: eventdata.DeviceRenamed): void {
+    public emitPublishAvailability(): void {
+        this.emitter.emit('publishAvailability');
+    }
+    public onPublishAvailability(key: ListenerKey, callback: () => void): void {
+        this.on('publishAvailability', callback, key);
+    }
+
+    public emitEntityRenamed(data: eventdata.EntityRenamed): void {
        this.emitter.emit('deviceRenamed', data);
    }
-    public onDeviceRenamed(key: ListenerKey, callback: (data: eventdata.DeviceRenamed) => void): void {
+    public onEntityRenamed(key: ListenerKey, callback: (data: eventdata.EntityRenamed) => void): void {
        this.on('deviceRenamed', callback, key);
    }

@@ -77,6 +84,13 @@ export default class EventBus {
        this.on('deviceJoined', callback, key);
    }

+    public emitEntityOptionsChanged(data: eventdata.EntityOptionsChanged): void {
+        this.emitter.emit('entityOptionsChanged', data);
+    }
+    public onEntityOptionsChanged(key: ListenerKey, callback: (data: eventdata.EntityOptionsChanged) => void): void {
+        this.on('entityOptionsChanged', callback, key);
+    }
+
    public emitDeviceLeave(data: eventdata.DeviceLeave): void {
        this.emitter.emit('deviceLeave', data);
    }
@@ -133,11 +147,11 @@ export default class EventBus {
        this.on('scenesChanged', callback, key);
    }

-    public emitReportingDisabled(data: eventdata.ReportingDisabled): void {
-        this.emitter.emit('reportingDisabled', data);
+    public emitReconfigure(data: eventdata.Reconfigure): void {
+        this.emitter.emit('reconfigure', data);
    }
-    public onReportingDisabled(key: ListenerKey, callback: (data: eventdata.ReportingDisabled) => void): void {
-        this.on('reportingDisabled', callback, key);
+    public onReconfigure(key: ListenerKey, callback: (data: eventdata.Reconfigure) => void): void {
+        this.on('reconfigure', callback, key);
    }

    public emitStateChange(data: eventdata.StateChange): void {
@@ -5,6 +5,12 @@ import * as settings from '../util/settings';
 import debounce from 'debounce';
 import bind from 'bind-decorator';

+const retrieveOnReconnect = [
+    {keys: ['state']},
+    {keys: ['brightness'], condition: (state: KeyValue): boolean => state.state === 'ON'},
+    {keys: ['color', 'color_temp'], condition: (state: KeyValue): boolean => state.state === 'ON'},
+];
+
 export default class Availability extends Extension {
    private timers: {[s: string]: NodeJS.Timeout} = {};
    private availabilityCache: {[s: string]: boolean} = {};
@@ -13,17 +19,14 @@ export default class Availability extends Extension {
    private pingQueueExecuting = false;

    private getTimeout(device: Device): number {
-        if (typeof device.settings.availability === 'object' && device.settings.availability?.timeout != null) {
-            return utils.minutes(device.settings.availability.timeout);
+        if (typeof device.options.availability === 'object' && device.options.availability?.timeout != null) {
+            return utils.minutes(device.options.availability.timeout);
        }

        const key = this.isActiveDevice(device) ? 'active' : 'passive';
-        const availabilitySettings = settings.get().availability;
-        if (typeof availabilitySettings === 'object' && availabilitySettings[key]?.timeout != null) {
-            return utils.minutes(availabilitySettings[key]?.timeout);
-        }
-
-        return key === 'active' ? utils.minutes(10) : utils.hours(25);
+        let value = settings.get().availability?.[key]?.timeout;
+        if (value == null) value = key == 'active' ? 10 : 1500;
+        return utils.minutes(value);
    }

    private isActiveDevice(device: Device): boolean {
@@ -31,9 +34,14 @@ export default class Availability extends Extension {
            device.zh.powerSource === 'Mains (single phase)';
    }

-    private isAvailable(device: Device): boolean {
-        const ago = Date.now() - device.zh.lastSeen;
-        return ago < this.getTimeout(device);
+    private isAvailable(entity: Device | Group): boolean {
+        if (entity.isDevice()) {
+            const ago = Date.now() - entity.zh.lastSeen;
+            return ago < this.getTimeout(entity);
+        } else {
+            return entity.membersDevices().length === 0 ||
+                entity.membersDevices().map((d) => this.availabilityCache[d.ieeeAddr]).includes(true);
+        }
    }

    private resetTimer(device: Device): void {
@@ -77,7 +85,7 @@ export default class Availability extends Extension {
                logger.debug(`Succesfully pinged '${device.name}' (attempt ${i + 1}/${attempts})`);
                break;
            } catch (error) {
-                logger.error(`Failed to ping '${device.name}' (attempt ${i + 1}/${attempts}, ${error.message})`);
+                logger.warn(`Failed to ping '${device.name}' (attempt ${i + 1}/${attempts}, ${error.message})`);
                // Try again in 3 seconds.
                const lastAttempt = i - 1 === attempts;
                !lastAttempt && await utils.sleep(3);
@@ -95,60 +103,78 @@ export default class Availability extends Extension {
    }

    override async start(): Promise<void> {
-        logger.warn('Using experimental new availability feature');
+        this.eventBus.onEntityRenamed(this, (data) => {
+            if (utils.isAvailabilityEnabledForEntity(data.entity, settings.get())) {
+                this.mqtt.publish(`${data.from}/availability`, null, {retain: true, qos: 0});
+                this.publishAvailability(data.entity, false, true);
+            }
+        });

-        this.eventBus.onDeviceRenamed(this, (data: eventdata.DeviceRenamed) =>
-            this.publishAvailability(data.device, false, true));
-        this.eventBus.onDeviceLeave(this, (data: eventdata.DeviceLeave) => clearTimeout(this.timers[data.ieeeAddr]));
-        this.eventBus.onDeviceAnnounce(this, (data: eventdata.DeviceAnnounce) => this.retrieveState(data.device));
+        this.eventBus.onDeviceRemoved(this, (data) => clearTimeout(this.timers[data.ieeeAddr]));
+        this.eventBus.onDeviceLeave(this, (data) => clearTimeout(this.timers[data.ieeeAddr]));
+        this.eventBus.onDeviceAnnounce(this, (data) => this.retrieveState(data.device));
        this.eventBus.onLastSeenChanged(this, this.onLastSeenChanged);
+        this.eventBus.onPublishAvailability(this, this.publishAvailabilityForAllEntities);
+        this.eventBus.onGroupMembersChanged(this, (data) => this.publishAvailability(data.group, false));
+        this.publishAvailabilityForAllEntities();
+    }

-        for (const device of this.zigbee.devices(false)) {
-            if (utils.isAvailabilityEnabledForDevice(device, settings.get())) {
+    @bind private publishAvailabilityForAllEntities(): void {
+        for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
+            if (utils.isAvailabilityEnabledForEntity(entity, settings.get())) {
                // Publish initial availablility
-                this.publishAvailability(device, true);
+                this.publishAvailability(entity, true, false, true);

-                this.resetTimer(device);
+                if (entity.isDevice()) {
+                    this.resetTimer(entity);

-                // If an active device is initially unavailable, ping it.
-                if (this.isActiveDevice(device) && !this.isAvailable(device)) {
-                    this.addToPingQueue(device);
+                    // If an active device is initially unavailable, ping it.
+                    if (this.isActiveDevice(entity) && !this.isAvailable(entity)) {
+                        this.addToPingQueue(entity);
+                    }
                }
            }
        }
    }

-    private publishAvailability(device: Device, logLastSeen: boolean, forcePublish=false): void {
-        if (logLastSeen) {
-            const ago = Date.now() - device.zh.lastSeen;
-            if (this.isActiveDevice(device)) {
-                logger.debug(`Active device '${device.name}' was last seen ` +
+    private publishAvailability(entity: Device | Group, logLastSeen: boolean,
+        forcePublish=false, skipGroups=false): void {
+        if (logLastSeen && entity.isDevice()) {
+            const ago = Date.now() - entity.zh.lastSeen;
+            if (this.isActiveDevice(entity)) {
+                logger.debug(`Active device '${entity.name}' was last seen ` +
                    `'${(ago / utils.minutes(1)).toFixed(2)}' minutes ago.`);
            } else {
                logger.debug(
-                    `Passive device '${device.name}' was last seen '${(ago / utils.hours(1)).toFixed(2)}' hours ago.`);
+                    `Passive device '${entity.name}' was last seen '${(ago / utils.hours(1)).toFixed(2)}' hours ago.`);
            }
        }

-        const available = this.isAvailable(device);
-        if (!forcePublish && this.availabilityCache[device.ieeeAddr] == available) {
+        const available = this.isAvailable(entity);
+        if (!forcePublish && this.availabilityCache[entity.ID] == available) {
            return;
        }

-        if (device.ieeeAddr in this.availabilityCache && available &&
-            this.availabilityCache[device.ieeeAddr] === false) {
-            logger.debug(`Device '${device.name}' reconnected`);
-            this.retrieveState(device);
+        if (entity.isDevice() && entity.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[entity.ieeeAddr] === false) {
+            logger.debug(`Device '${entity.name}' reconnected`);
+            this.retrieveState(entity);
        }

-        const topic = `${device.name}/availability`;
-        const payload = available ? 'online' : 'offline';
-        this.availabilityCache[device.ieeeAddr] = available;
+        const topic = `${entity.name}/availability`;
+        const payload = utils.availabilityPayload(available ? 'online' : 'offline', settings.get());
+        this.availabilityCache[entity.ID] = available;
        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+
+        if (!skipGroups && entity.isDevice()) {
+            this.zigbee.groups().filter((g) => g.hasMember(entity))
+                .filter((g) => utils.isAvailabilityEnabledForEntity(g, settings.get()))
+                .forEach((g) => this.publishAvailability(g, false, forcePublish));
+        }
    }

    @bind private onLastSeenChanged(data: eventdata.LastSeenChanged): void {
-        if (utils.isAvailabilityEnabledForDevice(data.device, settings.get())) {
+        if (utils.isAvailabilityEnabledForEntity(data.device, settings.get())) {
            // Remove from ping queue, not necessary anymore since we know the device is online.
            this.removeFromPingQueue(data.device);
            this.resetTimer(data.device);
@@ -157,8 +183,8 @@ export default class Availability extends Extension {
    }

    override async stop(): Promise<void> {
-        super.stop();
        Object.values(this.timers).forEach((t) => clearTimeout(t));
+        super.stop();
    }

    private retrieveState(device: Device): void {
@@ -168,17 +194,17 @@ export default class Availability extends Extension {
         */
        if (device.definition && !device.zh.interviewing && !this.retrieveStateDebouncers[device.ieeeAddr]) {
            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(async () => {
-                try {
-                    logger.debug(`Retrieving state of '${device.name}' after reconnect`);
-                    // Color and color temperature converters do both, only needs to be called once.
-                    const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
-                    for (const keys of keySet) {
-                        const converter = device.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
-                        await converter?.convertGet?.(device.endpoint(), keys[0],
-                            {message: this.state.get(device) || {}, mapped: device.definition});
-                    }
-                } catch (error) {
-                    logger.error(`Failed to read state of '${device.name}' after reconnect (${error.message})`);
+                logger.debug(`Retrieving state of '${device.name}' after reconnect`);
+                // Color and color temperature converters do both, only needs to be called once.
+                for (const item of retrieveOnReconnect) {
+                    if (item.condition && this.state.get(device) && !item.condition(this.state.get(device))) continue;
+                    const converter = device.definition.toZigbee.find((c) => c.key.find((k) => item.keys.includes(k)));
+                    await converter?.convertGet?.(device.endpoint(), item.keys[0],
+                        {message: this.state.get(device), mapped: device.definition})
+                        .catch((e) => {
+                            logger.error(`Failed to read state of '${device.name}' after reconnect (${e.message})`);
+                        });
+                    await utils.sleep(500);
                }
            }, utils.seconds(2));
        }
@@ -13,7 +13,8 @@ const legacyApi = settings.get().advanced.legacy_api;
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
-    'hvacThermostat', 'msTemperatureMeasurement'];
+    'hvacThermostat', 'msIlluminanceMeasurement', 'msTemperatureMeasurement', 'msRelativeHumidity',
+    'msSoilMoisture', 'msCO2'];

 // See zigbee-herdsman-converters
 const defaultBindGroup = {type: 'group_number', ID: 901, name: 'default_bind_group'};
@@ -67,7 +68,8 @@ const reportClusters: {[s: string]:
 type PollOnMessage = {
    cluster: {[s: string]: {type: string, data: KeyValue}[]}
    read: {cluster: string, attributes: string[], attributesForEndpoint?: (endpoint: zh.Endpoint) => Promise<string[]>}
-    manufacturerIDs: number[]
+    manufacturerIDs: number[],
+    manufacturerNames: string [],
 }[];

 const pollOnMessage: PollOnMessage = [
@@ -99,6 +101,11 @@ const pollOnMessage: PollOnMessage = [
            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+            zigbeeHersdman.Zcl.ManufacturerCode.TELINK,
+        ],
+        manufacturerNames: [
+            'GLEDOPTO',
+            'Trust International B.V.\u0000',
        ],
    },
    {
@@ -129,6 +136,11 @@ const pollOnMessage: PollOnMessage = [
            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+            zigbeeHersdman.Zcl.ManufacturerCode.TELINK,
+        ],
+        manufacturerNames: [
+            'GLEDOPTO',
+            'Trust International B.V.\u0000',
        ],
    },
    {
@@ -155,6 +167,11 @@ const pollOnMessage: PollOnMessage = [
            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+            zigbeeHersdman.Zcl.ManufacturerCode.TELINK,
+        ],
+        manufacturerNames: [
+            'GLEDOPTO',
+            'Trust International B.V.\u0000',
        ],
    },
 ];
@@ -417,7 +434,7 @@ export default class Bind extends Extension {
                }
            }

-            this.eventBus.emitReportingDisabled({device});
+            this.eventBus.emitReconfigure({device});
        }
    }

@@ -452,7 +469,8 @@ export default class Bind extends Extension {

            for (const endpoint of toPoll) {
                for (const poll of polls) {
-                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID) ||
+                    if ((!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID) &&
+                        !poll.manufacturerNames.includes(endpoint.getDevice().manufacturerName)) ||
                        !endpoint.supportsInputCluster(poll.read.cluster)) {
                        continue;
                    }
@@ -466,7 +484,12 @@ export default class Bind extends Extension {
                    const key = `${endpoint.getDevice().ieeeAddr}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
                    if (!this.pollDebouncers[key]) {
                        this.pollDebouncers[key] = debounce(async () => {
-                            await endpoint.read(poll.read.cluster, readAttrs);
+                            try {
+                                await endpoint.read(poll.read.cluster, readAttrs);
+                            } catch (error) {
+                                logger.error(`Failed to poll ${readAttrs} from ` +
+                                    `${this.zigbee.resolveEntity(endpoint.getDevice()).name}`);
+                            }
                        }, 1000);
                    }

@@ -10,6 +10,9 @@ import {detailedDiff} from 'deep-object-diff';
 import Extension from './extension';
 import Device from '../model/device';
 import Group from '../model/group';
+import data from '../util/data';
+import JSZip from 'jszip';
+import fs from 'fs';

 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);

@@ -38,8 +41,10 @@ export default class Bridge extends Extension {
            'group/rename': this.groupRename,
            'permit_join': this.permitJoin,
            'restart': this.restart,
+            'backup': this.backup,
            'touchlink/factory_reset': this.touchlinkFactoryReset,
            'touchlink/identify': this.touchlinkIdentify,
+            'install_code/add': this.installCodeAdd,
            'touchlink/scan': this.touchlinkScan,
            'health_check': this.healthCheck,
            'options': this.bridgeOptions,
@@ -64,7 +69,7 @@ export default class Bridge extends Extension {
        this.zigbee2mqttVersion = await utils.getZigbee2MQTTVersion();
        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();

-        this.eventBus.onDeviceRenamed(this, () => this.publishInfo());
+        this.eventBus.onEntityRenamed(this, () => this.publishInfo());
        this.eventBus.onGroupMembersChanged(this, () => this.publishGroups());
        this.eventBus.onDevicesChanged(this, () => this.publishDevices() && this.publishInfo());
        this.eventBus.onPermitJoinChanged(this, () => !this.zigbee.isStopping() && this.publishInfo());
@@ -157,7 +162,9 @@ export default class Bridge extends Extension {
        };
        cleanupDeleted(message.options, diff.deleted);

-        const newSettings = objectAssignDeep({}, diff.added, diff.updated, diff.deleted);
+        // objectAssignDeep requires object prototype which is missing from detailedDiff, therefore clone
+        const newSettings = objectAssignDeep({}, utils.clone(diff.added), utils.clone(diff.updated),
+            utils.clone(diff.deleted));

        // deep-object-diff converts arrays to objects, set original array back here
        const convertBackArray = (before: KeyValue, after: KeyValue): void => {
@@ -233,6 +240,28 @@ export default class Bridge extends Extension {
        return utils.getResponse(message, {}, null);
    }

+    @bind async backup(message: string | KeyValue): Promise<MQTTResponse> {
+        await this.zigbee.backup();
+        const dataPath = data.getPath();
+        const files = utils.getAllFiles(dataPath).map((f) => [f, f.substring(dataPath.length + 1)])
+            .filter((f) => !f[1].startsWith('log'));
+        const zip = new JSZip();
+        files.forEach((f) => zip.file(f[1], fs.readFileSync(f[0])));
+        const base64Zip = await zip.generateAsync({type: 'base64'});
+        return utils.getResponse(message, {zip: base64Zip}, null);
+    }
+
+    @bind async installCodeAdd(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        const value = typeof message === 'object' ? message.value : message;
+        await this.zigbee.addInstallCode(value);
+        logger.info('Successfully added new install code');
+        return utils.getResponse(message, {value}, null);
+    }
+
    @bind async permitJoin(message: KeyValue | string): Promise<MQTTResponse> {
        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
            throw new Error('Invalid payload');
@@ -392,13 +421,14 @@ export default class Bridge extends Extension {

        const ID = message.id;
        const entity = this.getEntity(entityType, ID);
-        const oldOptions = objectAssignDeep({}, cleanup(entity.settings));
+        const oldOptions = objectAssignDeep({}, cleanup(entity.options));
        settings.changeEntityOptions(ID, message.options);
-        const newOptions = cleanup(entity.settings);
+        const newOptions = cleanup(entity.options);
        await this.publishInfo();

        logger.info(`Changed config for ${entityType} ${ID}`);

+        this.eventBus.emitEntityOptionsChanged({from: oldOptions, to: newOptions, entity});
        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
    }

@@ -418,7 +448,7 @@ export default class Bridge extends Extension {
        await endpoint.configureReporting(message.cluster, [{
            attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
            maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
-        }]);
+        }], message.options);

        this.publishDevices();

@@ -447,16 +477,17 @@ export default class Bridge extends Extension {
        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
            message.homeassistant_rename : false;
        const entity = this.getEntity(entityType, from);
-        const oldFriendlyName = entity.settings.friendly_name;
+        const oldFriendlyName = entity.options.friendly_name;

        settings.changeFriendlyName(from, to);

        // Clear retained messages
        this.mqtt.publish(oldFriendlyName, '', {retain: true});

+        this.eventBus.emitEntityRenamed({entity: entity, homeAssisantRename, from: oldFriendlyName, to});
+
        if (entity instanceof Device) {
            this.publishDevices();
-            this.eventBus.emitDeviceRenamed({device: entity, homeAssisantRename, from: oldFriendlyName, to});
        } else {
            this.publishGroups();
            this.publishInfo();
@@ -565,7 +596,10 @@ export default class Bridge extends Extension {
        const payload = {
            version: this.zigbee2mqttVersion.version,
            commit: this.zigbee2mqttVersion.commitHash,
-            coordinator: this.coordinatorVersion,
+            coordinator: {
+                ieee_address: this.zigbee.firstCoordinatorEndpoint().getDevice().ieeeAddr,
+                ...this.coordinatorVersion,
+            },
            network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
            log_level: logger.getLevel(),
            permit_join: this.zigbee.getPermitJoin(),
@@ -645,6 +679,7 @@ export default class Bridge extends Extension {
                network_address: device.zh.networkAddress,
                supported: !!device.definition,
                friendly_name: device.name,
+                description: device.options.description,
                definition: this.getDefinitionPayload(device),
                power_source: device.zh.powerSource,
                software_build_id: device.zh.softwareBuildID,
@@ -666,6 +701,7 @@ export default class Bridge extends Extension {
            return {
                id: g.ID,
                friendly_name: g.ID === 901 ? 'default_bind_group' : g.name,
+                description: g.options.description,
                scenes: this.getScenes(g.zh),
                members: g.zh.members.map((e) => {
                    return {ieee_address: e.getDevice().ieeeAddr, endpoint: e.ID};
@@ -678,7 +714,7 @@ export default class Bridge extends Extension {

    getDefinitionPayload(device: Device): DefinitionPayload {
        if (!device.definition) return null;
-        let icon = device.settings.icon ? device.settings.icon : device.definition.icon;
+        let icon = device.options.icon ? device.options.icon : device.definition.icon;
        if (icon) {
            icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(device.zh.modelID));
            icon = icon.replace('${model}', utils.sanitizeImageParameter(device.definition.model));
@@ -688,7 +724,7 @@ export default class Bridge extends Extension {
            model: device.definition.model,
            vendor: device.definition.vendor,
            description: device.definition.description,
-            exposes: device.definition.exposes,
+            exposes: device.exposes(),
            supports_ota: !!device.definition.ota,
            options: device.definition.options,
            icon,
@@ -16,7 +16,7 @@ export default class Configure extends Extension {
    private topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
    private legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;

-    @bind private async onReportingDisabled(data: eventdata.ReportingDisabled): Promise<void> {
+    @bind private async onReconfigure(data: eventdata.Reconfigure): Promise<void> {
        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
        if (data.device.zh.meta?.hasOwnProperty('configured')) {
            delete data.device.zh.meta.configured;
@@ -79,13 +79,13 @@ export default class Configure extends Extension {
        this.eventBus.onDeviceInterview(this, (data) => this.configure(data.device, 'zigbee_event'));
        this.eventBus.onLastSeenChanged(this, (data) => this.configure(data.device, 'zigbee_event'));
        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
-        this.eventBus.onReportingDisabled(this, this.onReportingDisabled);
+        this.eventBus.onReconfigure(this, this.onReconfigure);
    }

    private async configure(device: Device, event: 'started' | 'zigbee_event' | 'reporting_disabled' | 'mqtt_message',
        force=false, thowError=false): Promise<void> {
        if (!force) {
-            if (!device.definition?.configure || device.zh.interviewing) {
+            if (!device.definition?.configure || !device.zh.interviewCompleted) {
                return;
            }

@@ -112,7 +112,8 @@ export default class Configure extends Extension {

        logger.info(`Configuring '${device.name}'`);
        try {
-            await device.definition.configure(device.zh, this.zigbee.firstCoordinatorEndpoint(), logger);
+            await device.definition.configure(device.zh, this.zigbee.firstCoordinatorEndpoint(), logger,
+                device.options);
            logger.info(`Successfully configured '${device.name}'`);
            device.zh.meta.configured = zhc.getConfigureKey(device.definition);
            device.zh.save();
@@ -6,7 +6,7 @@ abstract class Extension {
    protected eventBus: EventBus;
    protected enableDisableExtension: (enable: boolean, name: string) => Promise<void>;
    protected restartCallback: () => void;
-    protected addExtension: (extension: Extension) => void;
+    protected addExtension: (extension: Extension) => Promise<void>;

    /**
     * Besides intializing variables, the constructor should do nothing!
@@ -22,7 +22,7 @@ abstract class Extension {
     */
    constructor(zigbee: Zigbee, mqtt: MQTT, state: State, publishEntityState: PublishEntityState,
        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
-        restartCallback: () => void, addExtension: (extension: Extension) => void) {
+        restartCallback: () => void, addExtension: (extension: Extension) => Promise<void>) {
        this.zigbee = zigbee;
        this.mqtt = mqtt;
        this.state = state;
@@ -6,7 +6,7 @@ import Extension from './extension';
 export default class ExternalConverters extends Extension {
    constructor(zigbee: Zigbee, mqtt: MQTT, state: State, publishEntityState: PublishEntityState,
        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
-        restartCallback: () => void, addExtension: (extension: Extension) => void) {
+        restartCallback: () => void, addExtension: (extension: Extension) => Promise<void>) {
        super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);

        for (const definition of utils.getExternalConvertersDefinitions(settings.get())) {
@@ -11,7 +11,7 @@ import Extension from './extension';
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);

 export default class ExternalExtension extends Extension {
-    private requestLookup: {[s: string]: (message: KeyValue) => MQTTResponse};
+    private requestLookup: {[s: string]: (message: KeyValue) => Promise<MQTTResponse>};

    override async start(): Promise<void> {
        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
@@ -36,13 +36,13 @@ export default class ExternalExtension extends Extension {
        }
    }

-    @bind private removeExtension(message: KeyValue): MQTTResponse {
+    @bind private async removeExtension(message: KeyValue): Promise<MQTTResponse> {
        const {name} = message;
        const extensions = this.getListOfUserDefinedExtensions();
        const extensionToBeRemoved = extensions.find((e) => e.name === name);

        if (extensionToBeRemoved) {
-            this.enableDisableExtension(false, extensionToBeRemoved.name);
+            await this.enableDisableExtension(false, extensionToBeRemoved.name);
            const basePath = this.getExtensionsBasePath();
            const extensionFilePath = path.join(basePath, path.basename(name));
            fs.unlinkSync(extensionFilePath);
@@ -54,10 +54,10 @@ export default class ExternalExtension extends Extension {
        }
    }

-    @bind private saveExtension(message: KeyValue): MQTTResponse {
+    @bind private async saveExtension(message: KeyValue): Promise<MQTTResponse> {
        const {name, code} = message;
-        const ModuleConstructor = utils.loadModuleFromText(code) as Extension;
-        this.loadExtension(ModuleConstructor);
+        const ModuleConstructor = utils.loadModuleFromText(code) as typeof Extension;
+        await this.loadExtension(ModuleConstructor);
        const basePath = this.getExtensionsBasePath();
        /* istanbul ignore else */
        if (!fs.existsSync(basePath)) {
@@ -75,7 +75,7 @@ export default class ExternalExtension extends Extension {
        if (match && this.requestLookup[match[1].toLowerCase()]) {
            const message = utils.parseJSON(data.message, data.message) as KeyValue;
            try {
-                const response = this.requestLookup[match[1].toLowerCase()](message);
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
            } catch (error) {
                logger.error(`Request '${data.topic}' failed with error: '${error.message}'`);
@@ -85,11 +85,11 @@ export default class ExternalExtension extends Extension {
        }
    }

-    @bind private loadExtension(ConstructorClass: Extension): void {
-        this.enableDisableExtension(false, ConstructorClass.constructor.name);
+    @bind private async loadExtension(ConstructorClass: typeof Extension): Promise<void> {
+        await this.enableDisableExtension(false, ConstructorClass.name);
        // @ts-ignore
-        this.addExtension(new ConstructorClass(
-            this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus, settings, logger));
+        await this.addExtension(new ConstructorClass(this.zigbee, this.mqtt, this.state, this.publishEntityState,
+            this.eventBus, settings, logger));
    }

    private loadUserDefinedExtensions(): void {
@@ -1,5 +1,5 @@
 import http from 'http';
-import serveStatic from 'serve-static';
+import gzipStatic, {RequestHandler} from 'connect-gzip-static';
 import finalhandler from 'finalhandler';
 import logger from '../util/logger';
 import frontend from 'zigbee2mqtt-frontend';
@@ -17,18 +17,18 @@ import bind from 'bind-decorator';
 */
 export default class Frontend extends Extension {
    private mqttBaseTopic = settings.get().mqtt.base_topic;
-    private host = settings.get().frontend.host || '0.0.0.0';
-    private port = settings.get().frontend.port || 8080;
-    private authToken = settings.get().frontend.auth_token || false;
+    private host = settings.get().frontend.host;
+    private port = settings.get().frontend.port;
+    private authToken = settings.get().frontend.auth_token;
    private retainedMessages = new Map();
    private server: http.Server;
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    private fileServer: serveStatic.RequestHandler<any>;
+    private fileServer: RequestHandler;
    private wss: WebSocket.Server = null;

    constructor(zigbee: Zigbee, mqtt: MQTT, state: State, publishEntityState: PublishEntityState,
        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
-        restartCallback: () => void, addExtension: (extension: Extension) => void) {
+        restartCallback: () => void, addExtension: (extension: Extension) => Promise<void>) {
        super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);
        this.eventBus.onMQTTMessagePublished(this, this.onMQTTPublishMessage);
    }
@@ -37,14 +37,16 @@ export default class Frontend extends Extension {
        this.server = http.createServer(this.onRequest);
        this.server.on('upgrade', this.onUpgrade);

-        /* istanbul ignore next */ // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const options = {setHeaders: (res: any, path: any): void => {
-            if (path.endsWith('index.html')) {
-                res.setHeader('Cache-Control', 'no-store');
-            }
-        }};
-
-        this.fileServer = serveStatic(frontend.getPath(), options);
+        /* istanbul ignore next */
+        const options = {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            setHeaders: (res: any, path: string): void => {
+                if (path.endsWith('index.html')) {
+                    res.setHeader('Cache-Control', 'no-store');
+                }
+            },
+        };
+        this.fileServer = gzipStatic(frontend.getPath(), options);
        this.wss = new WebSocket.Server({noServer: true});
        this.wss.on('connection', this.onWebSocketConnection);

@@ -98,11 +100,7 @@ export default class Frontend extends Extension {
        }

        for (const device of this.zigbee.devices(false)) {
-            let payload: KeyValue = {};
-            if (this.state.exists(device)) {
-                payload = {...payload, ...this.state.get(device)};
-            }
-
+            const payload = this.state.get(device);
            const lastSeen = settings.get().advanced.last_seen;
            /* istanbul ignore if */
            if (lastSeen !== 'disable') {
@@ -13,17 +13,17 @@ const topicRegex =
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);

-const stateProperties: {[s: string]: (value: string, definition: zhc.Definition) => boolean} = {
+const stateProperties: {[s: string]: (value: string, exposes: zhc.DefinitionExpose[]) => boolean} = {
    'state': () => true,
-    'brightness': (value, definition) =>
-        !!definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
-    'color_temp': (value, definition) =>
-        !!definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
-    'color': (value, definition) =>
-        !!definition.exposes.find((e) => e.type === 'light' &&
+    'brightness': (value, exposes) =>
+        !!exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, exposes) =>
+        !!exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, exposes) =>
+        !!exposes.find((e) => e.type === 'light' &&
            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
-    'color_mode': (value, definition) =>
-        !!definition.exposes.find((e) => e.type === 'light' && (
+    'color_mode': (value, exposes) =>
+        !!exposes.find((e) => e.type === 'light' && (
            (e.features.find((f) => f.name === `color_${value}`)) ||
            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
 };
@@ -101,7 +101,7 @@ export default class Groups extends Extension {

    @bind async onStateChange(data: eventdata.StateChange): Promise<void> {
        const reason = 'groupOptimistic';
-        if (data.reason === reason) {
+        if (data.reason === reason || data.reason === 'publishCached') {
            return;
        }

@@ -123,17 +123,16 @@ export default class Groups extends Extension {
        if (Object.keys(payload).length) {
            const entity = data.entity;
            const groups = this.zigbee.groups().filter((g) => {
-                return g.settings && (!g.settings.hasOwnProperty('optimistic') || g.settings.optimistic);
+                return g.options && (!g.options.hasOwnProperty('optimistic') || g.options.optimistic);
            });

            if (entity instanceof Device) {
                for (const group of groups) {
                    if (group.zh.hasMember(entity.endpoint(endpointName)) &&
-                        !equals(this.lastOptimisticState[group.ID], payload)) {
-                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(group)) {
-                            this.lastOptimisticState[group.ID] = payload;
-                            await this.publishEntityState(group, payload, reason);
-                        }
+                        !equals(this.lastOptimisticState[group.ID], payload) &&
+                        this.shouldPublishPayloadForGroup(group, payload)) {
+                        this.lastOptimisticState[group.ID] = payload;
+                        await this.publishEntityState(group, payload, reason);
                    }
                }
            } else {
@@ -143,9 +142,10 @@ export default class Groups extends Extension {
                const groupsToPublish: Set<Group> = new Set();
                for (const member of entity.zh.members) {
                    const device = this.zigbee.resolveEntity(member.getDevice()) as Device;
+                    const exposes = device.exposes();
                    const memberPayload: KeyValue = {};
                    Object.keys(payload).forEach((key) => {
-                        if (stateProperties[key](payload[key], device.definition)) {
+                        if (stateProperties[key](payload[key], exposes)) {
                            memberPayload[key] = payload[key];
                        }
                    });
@@ -160,10 +160,9 @@ export default class Groups extends Extension {

                    await this.publishEntityState(device, memberPayload, reason);
                    for (const zigbeeGroup of groups) {
-                        if (zigbeeGroup.zh.hasMember(member)) {
-                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
-                                groupsToPublish.add(zigbeeGroup);
-                            }
+                        if (zigbeeGroup.zh.hasMember(member) &&
+                            this.shouldPublishPayloadForGroup(zigbeeGroup, payload)) {
+                            groupsToPublish.add(zigbeeGroup);
                        }
                    }
                }
@@ -175,12 +174,19 @@ export default class Groups extends Extension {
        }
    }

+    private shouldPublishPayloadForGroup(group: Group, payload: KeyValue): boolean {
+        if (group.options.off_state === 'last_member_state') return true;
+        if (!payload || payload.state !== 'OFF') return true;
+        if (this.areAllMembersOff(group)) return true;
+        return false;
+    }
+
    private areAllMembersOff(group: Group): boolean {
        for (const member of group.zh.members) {
            const device = this.zigbee.resolveEntity(member.getDevice());
            if (this.state.exists(device)) {
                const state = this.state.get(device);
-                if (state && state.state === 'ON') {
+                if (state.state === 'ON') {
                    return false;
                }
            }
@@ -286,6 +292,7 @@ export default class Groups extends Extension {
            groupKey, deviceKey, skipDisableReporting, resolvedEntityEndpoint,
        } = parsed;
        const message = utils.parseJSON(data.message, data.message);
+        let changedGroups: Group[] = [];

        const responseData: KeyValue = {device: deviceKey};
        if (groupKey) {
@@ -314,6 +321,7 @@ export default class Groups extends Extension {
                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
                    await resolvedEntityEndpoint.addToGroup(resolvedEntityGroup.zh);
                    settings.addDeviceToGroup(resolvedEntityGroup.ID.toString(), keys);
+                    changedGroups.push(resolvedEntityGroup);

                    /* istanbul ignore else */
                    if (settings.get().advanced.legacy_api) {
@@ -327,6 +335,7 @@ export default class Groups extends Extension {
                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
                    await resolvedEntityEndpoint.removeFromGroup(resolvedEntityGroup.zh);
                    settings.removeDeviceFromGroup(resolvedEntityGroup.ID.toString(), keys);
+                    changedGroups.push(resolvedEntityGroup);

                    /* istanbul ignore else */
                    if (settings.get().advanced.legacy_api) {
@@ -338,6 +347,7 @@ export default class Groups extends Extension {
                    }
                } else { // remove_all
                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    changedGroups = this.zigbee.groups().filter((g) => g.zh.members.includes(resolvedEntityEndpoint));
                    await resolvedEntityEndpoint.removeFromAllGroups();
                    for (const settingsGroup of settings.getGroups()) {
                        settings.removeDeviceFromGroup(settingsGroup.ID.toString(), keys);
@@ -366,8 +376,10 @@ export default class Groups extends Extension {
        if (error) {
            logger.error(error);
        } else {
-            this.eventBus.emitGroupMembersChanged({
-                group: resolvedEntityGroup, action: type, endpoint: resolvedEntityEndpoint, skipDisableReporting});
+            for (const group of changedGroups) {
+                this.eventBus.emitGroupMembersChanged({
+                    group, action: type, endpoint: resolvedEntityEndpoint, skipDisableReporting});
+            }
        }
    }
 }
@@ -51,7 +51,7 @@ export default class BridgeLegacy extends Extension {
        try {
            const entity = settings.getDevice(message);
            assert(entity, `Entity '${message}' does not exist`);
-            settings.whitelistDevice(entity.ID.toString());
+            settings.addDeviceToPasslist(entity.ID.toString());
            logger.info(`Whitelisted '${entity.friendly_name}'`);
            this.mqtt.publish(
                'bridge/log',
@@ -217,7 +217,7 @@ export default class BridgeLegacy extends Extension {
            logger.info(`Successfully renamed - ${from} to ${to} `);
            const entity = this.zigbee.resolveEntity(to);
            if (entity.isDevice()) {
-                this.eventBus.emitDeviceRenamed({homeAssisantRename: false, from, to, device: entity});
+                this.eventBus.emitEntityRenamed({homeAssisantRename: false, from, to, entity});
            }

            this.mqtt.publish(
@@ -330,13 +330,13 @@ export default class BridgeLegacy extends Extension {
        } catch (error) {
            logger.error(`Failed to ${lookup[action][2]} ${entity.name} (${error})`);
            // eslint-disable-next-line
-            logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
+            logger.error(`See https://www.zigbee2mqtt.io/guide/usage/mqtt_topics_and_messages.html#zigbee2mqtt-bridge-request for more info`);

            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
        }

        if (action === 'ban') {
-            settings.banDevice(ieeeAddr);
+            settings.blockDevice(ieeeAddr);
        }
    }

@@ -116,7 +116,7 @@ export default class Report extends Extension {
                device.zh.meta.reporting = reportKey;
            } else {
                delete device.zh.meta.reporting;
-                this.eventBus.emitReportingDisabled({device});
+                this.eventBus.emitReconfigure({device});
            }

            this.eventBus.emitDevicesChanged();
@@ -244,6 +244,7 @@ export default class NetworkMap extends Extension {
            } catch (error) {
                failed.get(device).push('lqi');
                logger.error(`Failed to execute LQI for '${device.name}'`);
+                logger.debug(error.stack);
            }

            if (includeRoutes) {
@@ -267,7 +268,7 @@ export default class NetworkMap extends Extension {
                model: device.definition.model,
                vendor: device.definition.vendor,
                description: device.definition.description,
-                supports: Array.from(new Set((device.definition.exposes).map((e) => {
+                supports: Array.from(new Set((device.exposes()).map((e) => {
                    return e.hasOwnProperty('name') ? e.name :
                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
                }))).join(', '),
@@ -7,7 +7,7 @@ import Extension from './extension';
 export default class OnEvent extends Extension {
    override async start(): Promise<void> {
        for (const device of this.zigbee.devices(false)) {
-            await this.callOnEvent(device, 'start', {});
+            this.callOnEvent(device, 'start', {});
        }

        this.eventBus.onDeviceMessage(this, (data) => this.callOnEvent(data.device, 'message', this.convertData(data)));
@@ -19,6 +19,13 @@ export default class OnEvent extends Extension {
            (data) => this.callOnEvent(data.device, 'deviceAnnounce', this.convertData(data)));
        this.eventBus.onDeviceNetworkAddressChanged(this,
            (data) => this.callOnEvent(data.device, 'deviceNetworkAddressChanged', this.convertData(data)));
+        this.eventBus.onEntityOptionsChanged(this,
+            (data) => {
+                if (data.entity.isDevice()) {
+                    this.callOnEvent(data.entity, 'deviceOptionsChanged', data)
+                        .then(() => this.eventBus.emitDevicesChanged());
+                }
+            });
    }

    private convertData(data: KeyValue): KeyValue {
@@ -33,10 +40,11 @@ export default class OnEvent extends Extension {
    }

    private async callOnEvent(device: Device, type: string, data: KeyValue): Promise<void> {
-        zhc.onEvent(type, data, device.zh);
+        const state = this.state.get(device);
+        zhc.onEvent(type, data, device.zh, device.options, state);

        if (device.definition?.onEvent) {
-            await device.definition.onEvent(type, data, device.zh, device.settings);
+            await device.definition.onEvent(type, data, device.zh, device.options, state);
        }
    }
 }
@@ -3,9 +3,24 @@ import logger from '../util/logger';
 import stringify from 'json-stable-stringify-without-jsonify';
 import utils from '../util/utils';
 import tradfriOTA from 'zigbee-herdsman-converters/lib/ota/tradfri';
+import zigbeeOTA from 'zigbee-herdsman-converters/lib/ota/zigbeeOTA';
 import Extension from './extension';
 import bind from 'bind-decorator';
 import Device from '../model/device';
+import dataDir from '../util/data';
+import * as URI from 'uri-js';
+import path from 'path';
+
+function isValidUrl(url: string): boolean {
+    let parsed;
+    try {
+        parsed = URI.parse(url);
+    } catch (_) {
+        // istanbul ignore next
+        return false;
+    }
+    return parsed.scheme === 'http' || parsed.scheme === 'https';
+}

 type UpdateState = 'updating' | 'idle' | 'available';
 interface UpdatePayload {
@@ -24,40 +39,61 @@ export default class OTAUpdate extends Extension {
    override async start(): Promise<void> {
        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
        this.eventBus.onDeviceMessage(this, this.onZigbeeEvent);
-        if (settings.get().advanced.ikea_ota_use_test_url) {
+        if (settings.get().ota.ikea_ota_use_test_url) {
            tradfriOTA.useTestURL();
        }

+        // Let zigbeeOTA module know if the override index file is provided
+        let overrideOTAIndex = settings.get().ota.zigbee_ota_override_index_location;
+        if (overrideOTAIndex) {
+            // If the file name is not a full path, then treat it as a relative to the data directory
+            if (!isValidUrl(overrideOTAIndex) && !path.isAbsolute(overrideOTAIndex)) {
+                overrideOTAIndex = dataDir.joinPath(overrideOTAIndex);
+            }
+
+            zigbeeOTA.useIndexOverride(overrideOTAIndex);
+        }
+
+        // In order to support local firmware files we need to let zigbeeOTA know where the data directory is
+        zigbeeOTA.setDataDir(dataDir.getPath());
+
+        // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
+        // remove them.
        for (const device of this.zigbee.devices(false)) {
-            // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
-            // remove them.
            this.removeProgressAndRemainingFromState(device);
        }
    }

    private removeProgressAndRemainingFromState(device: Device): void {
-        delete this.state.get(device)?.update?.progress;
-        delete this.state.get(device)?.update?.remaining;
+        delete this.state.get(device).update?.progress;
+        delete this.state.get(device).update?.remaining;
    }

    @bind private async onZigbeeEvent(data: eventdata.DeviceMessage): Promise<void> {
-        if (settings.get().ota.disable_automatic_update_check) return;
-        if (data.type !== 'commandQueryNextImageRequest' || !data.device.definition) return;
+        if (data.type !== 'commandQueryNextImageRequest' || !data.device.definition ||
+            this.inProgress.has(data.device.ieeeAddr)) return;
        logger.debug(`Device '${data.device.name}' requested OTA`);

-        const supportsOTA = data.device.definition.hasOwnProperty('ota');
-        if (supportsOTA) {
+        const automaticOTACheckDisabled = settings.get().ota.disable_automatic_update_check;
+        let supportsOTA = data.device.definition.hasOwnProperty('ota');
+        if (supportsOTA && !automaticOTACheckDisabled) {
            // When a device does a next image request, it will usually do it a few times after each other
            // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
            // each time, so this interval can be set by the user. The default is 1,440 minutes (one day).
            const updateCheckInterval = settings.get().ota.update_check_interval * 1000 * 60;
            const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > updateCheckInterval : true;
-            if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
+            if (!check) return;

            this.lastChecked[data.device.ieeeAddr] = Date.now();
-            const available = await data.device.definition.ota.isUpdateAvailable(
-                data.device.zh, logger, data.data);
+            let available = false;
+            try {
+                available = await data.device.definition.ota.isUpdateAvailable(data.device.zh, logger, data.data);
+            } catch (e) {
+                supportsOTA = false;
+                logger.debug(`Failed to check if update available for '${data.device.name}' (${e.message})`);
+            }
+
            const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
            this.publishEntityState(data.device, payload);

@@ -76,28 +112,19 @@ export default class OTAUpdate extends Extension {
            }
        }

-        // Respond to the OTA request:
-        // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
-        // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
+        // Respond to the OTA request: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
        const endpoint = data.device.zh.endpoints.find((e) => e.supportsOutputCluster('genOta'));
        if (endpoint) {
            // Some devices send OTA requests without defining OTA cluster as input cluster.
-            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: 0x98});
        }
    }

-    private async readSoftwareBuildIDAndDateCode(device: Device, update: boolean):
+    private async readSoftwareBuildIDAndDateCode(device: Device, sendWhen: 'active' | 'immediate'):
        Promise<{softwareBuildID: string, dateCode: string}> {
        try {
            const endpoint = device.zh.endpoints.find((e) => e.supportsInputCluster('genBasic'));
-            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
-
-            if (update) {
-                device.zh.softwareBuildID = result.swBuildId;
-                device.zh.dateCode = result.dateCode;
-                device.zh.save();
-            }
-
+            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId'], {sendWhen});
            return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
        } catch (e) {
            return null;
@@ -224,16 +251,16 @@ export default class OTAUpdate extends Extension {
                        }
                    };

-                    const from_ = await this.readSoftwareBuildIDAndDateCode(device, false);
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(device, 'immediate');
                    await device.definition.ota.updateToLatest(device.zh, logger, onProgress);
-                    const to = await this.readSoftwareBuildIDAndDateCode(device, true);
-                    const [fromS, toS] = [stringify(from_), stringify(to)];
-                    const msg = `Finished update of '${device.name}'` +
-                        (to ? `, from '${fromS}' to '${toS}'` : ``);
-                    logger.info(msg);
+                    logger.info(`Finished update of '${device.name}'`);
+                    this.eventBus.emitReconfigure({device});
                    this.removeProgressAndRemainingFromState(device);
                    const payload = this.getEntityPublishPayload('idle');
                    this.publishEntityState(device, payload);
+                    const to = await this.readSoftwareBuildIDAndDateCode(device, 'active');
+                    const [fromS, toS] = [stringify(from_), stringify(to)];
+                    logger.info(`Device '${device.name}' was updated from '${fromS}' to '${toS}'`);
                    responseData.from = from_ ? utils.toSnakeCase(from_) : null;
                    responseData.to = to ? utils.toSnakeCase(to) : null;
                    this.eventBus.emitDevicesChanged();
@@ -10,7 +10,7 @@ import Device from '../model/device';
 import bind from 'bind-decorator';

 const topicRegex = new RegExp(`^(.+?)(?:/(${utils.endpointNames.join('|')}|\\d+))?/(get|set)(?:/(.+))?`);
-const propertyEndpointRegex = new RegExp(`^(.*)_(${utils.endpointNames.join('|')}|\\d+)$`);
+const propertyEndpointRegex = new RegExp(`^(.*?)_(${utils.endpointNames.join('|')})$`);
 const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
 const sceneConverterKeys = ['scene_store', 'scene_add', 'scene_remove', 'scene_remove_all'];

@@ -91,7 +91,7 @@ export default class Publish extends Extension {
        // Only do this when the retrieve_state option is enabled for this device.
        // retrieve_state == decprecated
        if (re instanceof Device && result && result.hasOwnProperty('readAfterWriteTime') &&
-            re.settings.retrieve_state
+            re.options.retrieve_state
        ) {
            setTimeout(() => converter.convertGet(target, key, meta), result.readAfterWriteTime);
        }
@@ -128,18 +128,14 @@ export default class Publish extends Extension {

        // Get entity details
        const definition = re instanceof Device ? re.definition : re.membersDefinitions();
-        if (definition == null) {
-            logger.error(`Cannot publish to unsupported device '${re.name}'`);
-            return;
-        }
        const target = re instanceof Group ? re.zh : re.endpoint(parsedTopic.endpoint);
        if (target == null) {
            logger.error(`Device '${re.name}' has no endpoint '${parsedTopic.endpoint}'`);
            return;
        }
        const device = re instanceof Device ? re.zh : null;
-        const entitySettings = re.settings;
-        const entityState = this.state.get(re) || {};
+        const entitySettings = re.options;
+        const entityState = this.state.get(re);
        const membersState = re instanceof Group ?
            Object.fromEntries(re.zh.members.map((e) => [e.getDevice().ieeeAddr,
                this.state.get(this.zigbee.resolveEntity(e.getDevice().ieeeAddr))])) : null;
@@ -149,8 +145,11 @@ export default class Publish extends Extension {
                const c = new Set(definition.map((d) => d.toZigbee).flat());
                if (c.size == 0) converters = defaultGroupConverters;
                else converters = Array.from(c);
-            } else {
+            } else if (definition) {
                converters = definition.toZigbee;
+            } else {
+                converters = [zigbeeHerdsmanConverters.toZigbeeConverters.read,
+                    zigbeeHerdsmanConverters.toZigbeeConverters.write];
            }
        }

@@ -216,7 +215,7 @@ export default class Publish extends Extension {
            }

            if (!converter) {
-                logger.error(`No converter available for '${key}' (${message[key]})`);
+                logger.error(`No converter available for '${key}' (${stringify(message[key])})`);
                continue;
            }

@@ -257,7 +256,8 @@ export default class Publish extends Extension {
                        }

                        // filter out attribute listed in filtered_optimistic
-                        entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
+                        utils.filterProperties(entitySettings.filtered_optimistic, msg);
+
                        addToToPublish(re, msg);
                    }

@@ -4,6 +4,7 @@ import debounce from 'debounce';
 import Extension from './extension';
 import stringify from 'json-stable-stringify-without-jsonify';
 import bind from 'bind-decorator';
+import utils from '../util/utils';

 type DebounceFunction = (() => void) & { clear(): void; } & { flush(): void; };

@@ -24,7 +25,7 @@ export default class Receive extends Extension {
         */
        if (data.entity.isDevice() && this.debouncers[data.entity.ieeeAddr] &&
            data.stateChangeReason !== 'publishDebounce' && data.stateChangeReason !== 'lastSeenChanged') {
-            for (const key of Object.keys(data.message)) {
+            for (const key of Object.keys(data.payload)) {
                delete this.debouncers[data.entity.ieeeAddr].payload[key];
            }
        }
@@ -76,7 +77,8 @@ export default class Receive extends Extension {
                logger.warn(
                    `Received message from unsupported device with Zigbee model '${data.device.zh.modelID}' ` +
                    `and manufacturer name '${data.device.zh.manufacturerName}'`);
-                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+                // eslint-disable-next-line max-len
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html`);
            }

            return false;
@@ -89,35 +91,24 @@ export default class Receive extends Extension {
        /* istanbul ignore next */
        if (!data.device) return;

-        /**
-         * Handling of re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The network address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the network address of the end device.
-         */
-        if (data.device.isXiaomi() && data.device.zh.type === 'Router' && data.groupID) {
-            logger.debug('Handling re-transmitted Xiaomi message');
-            data = {...data, device: this.zigbee.deviceByNetworkAddress(data.groupID)};
+        if (!this.shouldProcess(data)) {
+            utils.publishLastSeen({device: data.device, reason: 'messageEmitted'},
+                settings.get(), true, this.publishEntityState);
+            return;
        }

-        if (!this.shouldProcess(data)) return;
-
        const converters = data.device.definition.fromZigbee.filter((c) => {
            const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
            return c.cluster === data.cluster && type;
        });

        // Check if there is an available converter, genOta messages are not interesting.
-        const ignoreClusters: (string | number)[] = ['genOta', 'genTime', 'genBasic'];
+        const ignoreClusters: (string | number)[] = ['genOta', 'genTime', 'genBasic', 'genPollCtrl'];
        if (converters.length == 0 && !ignoreClusters.includes(data.cluster)) {
            logger.debug(`No converter available for '${data.device.definition.model}' with ` +
                `cluster '${data.cluster}' and type '${data.type}' and data '${stringify(data.data)}'`);
+            utils.publishLastSeen({device: data.device, reason: 'messageEmitted'},
+                settings.get(), true, this.publishEntityState);
            return;
        }

@@ -137,9 +128,9 @@ export default class Receive extends Extension {
            }

            // Check if we have to debounce
-            if (data.device.settings.debounce) {
-                this.publishDebounce(data.device, payload, data.device.settings.debounce,
-                    data.device.settings.debounce_ignore);
+            if (data.device.options.debounce) {
+                this.publishDebounce(data.device, payload, data.device.options.debounce,
+                    data.device.options.debounce_ignore);
            } else {
                this.publishEntityState(data.device, payload);
            }
@@ -150,18 +141,21 @@ export default class Receive extends Extension {
        for (const converter of converters) {
            try {
                const converted = await converter.convert(
-                    data.device.definition, data, publish, data.device.settings, meta);
+                    data.device.definition, data, publish, data.device.options, meta);
                if (converted) {
                    payload = {...payload, ...converted};
                }
-            } catch (error) {
-                // istanbul ignore next
+            } catch (error) /* istanbul ignore next */ {
                logger.error(`Exception while calling fromZigbee converter: ${error.message}}`);
+                logger.debug(error.stack);
            }
        }

        if (Object.keys(payload).length) {
            publish(payload);
+        } else {
+            utils.publishLastSeen({device: data.device, reason: 'messageEmitted'},
+                settings.get(), true, this.publishEntityState);
        }
    }
 }
@@ -5,16 +5,20 @@ import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
 export default class Device {
    public zh: zh.Device;
    private _definition: zhc.Definition;
+    private _definitionModelID: string;

    get ieeeAddr(): string {return this.zh.ieeeAddr;}
    get ID(): string {return this.zh.ieeeAddr;}
-    get settings(): DeviceSettings {return {...settings.get().device_options, ...settings.getDevice(this.ieeeAddr)};}
+    get options(): DeviceOptions {return {...settings.get().device_options, ...settings.getDevice(this.ieeeAddr)};}
    get name(): string {
-        return this.zh.type === 'Coordinator' ? 'Coordinator' : this.settings?.friendly_name || this.ieeeAddr;
+        return this.zh.type === 'Coordinator' ? 'Coordinator' : this.options?.friendly_name || this.ieeeAddr;
    }
    get definition(): zhc.Definition {
-        if (!this._definition && !this.zh.interviewing) {
+        // Some devices can change modelID, reconsider the definition in that case.
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/3016
+        if (!this.zh.interviewing && (!this._definition || this._definitionModelID !== this.zh.modelID)) {
            this._definition = zigbeeHerdsmanConverters.findByDevice(this.zh);
+            this._definitionModelID = this.zh.modelID;
        }
        return this._definition;
    }
@@ -23,6 +27,15 @@ export default class Device {
        this.zh = device;
    }

+    exposes(): zhc.DefinitionExpose[] {
+        /* istanbul ignore if */
+        if (typeof this.definition.exposes == 'function') {
+            return this.definition.exposes(this.zh, this.options);
+        } else {
+            return this.definition.exposes;
+        }
+    }
+
    ensureInSettings(): void {
        if (this.zh.type !== 'Coordinator' && !settings.getDevice(this.zh.ieeeAddr)) {
            settings.addDevice(this.zh.ieeeAddr);
@@ -58,13 +71,6 @@ export default class Device {
        return name === 'default' ? null : name;
    }

-    isXiaomi(): boolean {
-        const xiaomiManufacturerID = [4151, 4447];
-        /* istanbul ignore next */
-        return this.zh.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(this.zh.manufacturerID) &&
-            (!this.zh.manufacturerName || !this.zh.manufacturerName.startsWith('Trust'));
-    }
-
    isIkeaTradfri(): boolean {return this.zh.manufacturerID === 4476;}

    isDevice(): this is Device {return true;}
@@ -4,13 +4,23 @@ import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';

 export default class Group {
    public zh: zh.Group;
+    private resolveDevice: (ieeeAddr: string) => Device;

    get ID(): number {return this.zh.groupID;}
-    get settings(): GroupSettings {return settings.getGroup(this.ID);}
-    get name(): string {return this.settings?.friendly_name || this.ID.toString();}
+    get options(): GroupOptions {return {...settings.getGroup(this.ID)};}
+    get name(): string {return this.options?.friendly_name || this.ID.toString();}

-    constructor(group: zh.Group) {
+    constructor(group: zh.Group, resolveDevice: (ieeeAddr: string) => Device) {
        this.zh = group;
+        this.resolveDevice = resolveDevice;
+    }
+
+    hasMember(device: Device): boolean {
+        return !!device.zh.endpoints.find((e) => this.zh.members.includes(e));
+    }
+
+    membersDevices(): Device[] {
+        return this.zh.members.map((e) => this.resolveDevice(e.getDevice().ieeeAddr)).filter((d) => d);
    }

    membersDefinitions(): zhc.Definition[] {
@@ -10,6 +10,10 @@ export default class MQTT {
    private connectionTimer: NodeJS.Timeout;
    private client: mqtt.MqttClient;
    private eventBus: EventBus;
+    private initialConnect = true;
+    private republishRetainedTimer: NodeJS.Timer;
+    private retainedMessages: {[s: string]: {payload: string, options: MQTTOptions,
+        skipLog: boolean, skipReceive: boolean, topic: string, base: string}} = {};

    constructor(eventBus: EventBus) {
        this.eventBus = eventBus;
@@ -22,7 +26,7 @@ export default class MQTT {
        const options: mqtt.IClientOptions = {
            will: {
                topic: `${settings.get().mqtt.base_topic}/bridge/state`,
-                payload: 'offline',
+                payload: utils.availabilityPayload('offline', settings.get()),
                retain: settings.get().mqtt.force_disable_retain ? false : true,
                qos: 1,
            },
@@ -69,6 +73,9 @@ export default class MQTT {

        return new Promise((resolve, reject) => {
            this.client = mqtt.connect(mqttSettings.server, options);
+            // @ts-ignore https://github.com/Koenkk/zigbee2mqtt/issues/9822
+            this.client.stream.setMaxListeners(0);
+            this.eventBus.onPublishAvailability(this, () => this.publishStateOnline());

            const onConnect = this.onConnect;
            this.client.on('connect', async () => {
@@ -76,7 +83,10 @@ export default class MQTT {
                resolve();
            });

-            this.client.on('error', (err) => reject(err));
+            this.client.on('error', (err) => {
+                logger.error(`MQTT error: ${err.message}`);
+                reject(err);
+            });
            this.client.on('message', this.onMessage);
        });
    }
@@ -91,13 +101,30 @@ export default class MQTT {
        }, utils.seconds(10));

        logger.info('Connected to MQTT server');
+        await this.publishStateOnline();
+
+        if (!this.initialConnect) {
+            this.republishRetainedTimer = setTimeout(() => {
+                // Republish retained messages in case MQTT broker does not persist them.
+                // https://github.com/Koenkk/zigbee2mqtt/issues/9629
+                Object.values(this.retainedMessages).forEach((e) =>
+                    this.publish(e.topic, e.payload, e.options, e.base, e.skipLog, e.skipReceive));
+            }, 2000);
+        }
+
+        this.initialConnect = false;
        this.subscribe(`${settings.get().mqtt.base_topic}/#`);
-        await this.publish('bridge/state', 'online', {retain: true, qos: 0});
+    }
+
+    async publishStateOnline(): Promise<void> {
+        await this.publish('bridge/state', utils.availabilityPayload('online', settings.get()), {retain: true, qos: 0});
    }

    async disconnect(): Promise<void> {
        clearTimeout(this.connectionTimer);
-        await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
+        await this.publish('bridge/state', utils.availabilityPayload('offline', settings.get()),
+            {retain: true, qos: 0});
+        this.eventBus.removeListeners(this);
        logger.info('Disconnecting from MQTT server');
        this.client.end();
    }
@@ -112,6 +139,11 @@ export default class MQTT {
            logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
            this.eventBus.emitMQTTMessage({topic, message: message + ''});
        }
+
+        if (this.republishRetainedTimer && topic == `${settings.get().mqtt.base_topic}/bridge/state`) {
+            clearTimeout(this.republishRetainedTimer);
+            this.republishRetainedTimer = null;
+        }
    }

    isConnected(): boolean {
@@ -128,6 +160,15 @@ export default class MQTT {
            this.publishedTopics.add(topic);
        }

+        if (options.retain) {
+            if (payload) {
+                this.retainedMessages[topic] =
+                    {payload, options, skipReceive, skipLog, topic: topic.substring(base.length + 1), base};
+            } else {
+                delete this.retainedMessages[topic];
+            }
+        }
+
        this.eventBus.emitMQTTMessagePublished({topic, payload, options: {...defaultOptions, ...options}});

        if (!this.isConnected()) {
@@ -1,15 +1,16 @@
 import logger from './util/logger';
 import data from './util/data';
 import * as settings from './util/settings';
+import utils from './util/utils';
 import fs from 'fs';
 import objectAssignDeep from 'object-assign-deep';

 const saveInterval = 1000 * 60 * 5; // 5 minutes

 const dontCacheProperties = [
-    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
-    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
-    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];

 class State {
@@ -68,23 +69,20 @@ class State {
    }

    get(entity: Group | Device): KeyValue {
-        return this.state[entity.ID];
+        return this.state[entity.ID] || {};
    }

    set(entity: Group | Device, update: KeyValue, reason: string=null): KeyValue {
        const fromState = this.state[entity.ID] || {};
        const toState = objectAssignDeep({}, fromState, update);
-        const result = {...toState};
+        const newCache = {...toState};
+        const entityDontCacheProperties = entity.options.filtered_cache || [];

-        for (const property of Object.keys(toState)) {
-            if (dontCacheProperties.find((p) => property.match(p))) {
-                delete toState[property];
-            }
-        }
+        utils.filterProperties(dontCacheProperties.concat(entityDontCacheProperties), newCache);

-        this.state[entity.ID] = toState;
+        this.state[entity.ID] = newCache;
        this.eventBus.emitStateChange({entity, from: fromState, to: toState, reason, update});
-        return result;
+        return toState;
    }

    remove(ID: string | number): void {
@@ -42,7 +42,7 @@ declare global {
    // Types
    interface MQTTResponse {data: KeyValue, status: 'error' | 'ok', error?: string, transaction?: string}
    interface MQTTOptions {qos?: mqtt.QoS, retain?: boolean, properties?: {messageExpiryInterval: number}}
-    type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged';
+    type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged' | 'publishCached';
    type PublishEntityState = (entity: Device | Group, payload: KeyValue,
        stateChangeReason?: StateChangeReason) => Promise<void>;
    type RecursivePartial<T> = {[P in keyof T]?: RecursivePartial<T[P]>;};
@@ -95,12 +95,12 @@ declare global {
        }

        interface DefinitionExposeFeature {name: string, endpoint?: string,
-            property: string, value_max?: number, value_min?: number,
+            property: string, value_max?: number, value_min?: number, unit?: string,
            value_off?: string, value_on?: string, value_step?: number, values: string[], access: number}

        interface DefinitionExpose {
            type: string, name?: string, features?: DefinitionExposeFeature[],
-            endpoint?: string, values?: string[], value_off?: string, value_on?: string,
+            endpoint?: string, values?: string[], value_off?: string, value_on?: string, value_step?: number,
            access: number, property: string, unit?: string,
            value_min?: number, value_max?: number}

@@ -114,9 +114,11 @@ declare global {
            description: string
            options: zhc.DefinitionExpose[],
            vendor: string
-            exposes: DefinitionExpose[]
-            configure?: (device: zh.Device, coordinatorEndpoint: zh.Endpoint, logger: Logger) => Promise<void>;
-            onEvent?: (type: string, data: KeyValue, device: zh.Device, settings: KeyValue) => Promise<void>;
+            exposes: DefinitionExpose[] | ((device: zh.Device, options: KeyValue) => DefinitionExpose[])
+            configure?: (device: zh.Device, coordinatorEndpoint: zh.Endpoint, logger: Logger,
+                options?: DeviceOptions) => Promise<void>;
+            onEvent?: (type: string, data: KeyValue, device: zh.Device,
+                settings: KeyValue, state: KeyValue) => Promise<void>;
            ota?: {
                isUpdateAvailable: (device: zh.Device, logger: Logger, data?: KeyValue) => Promise<boolean>;
                updateToLatest: (device: zh.Device, logger: Logger,
@@ -129,23 +131,26 @@ declare global {
    }

    namespace eventdata {
-        type DeviceRenamed = { device: Device, homeAssisantRename: boolean, from: string, to: string };
+        type EntityRenamed = { entity: Device | Group, homeAssisantRename: boolean, from: string, to: string };
        type DeviceRemoved = { ieeeAddr: string, name: string };
        type MQTTMessage = { topic: string, message: string };
        type MQTTMessagePublished = { topic: string, payload: string, options: {retain: boolean, qos: number} };
        type StateChange = {
            entity: Device | Group, from: KeyValue, to: KeyValue, reason: string | null, update: KeyValue };
        type PermitJoinChanged = ZHEvents.PermitJoinChangedPayload;
-        type LastSeenChanged = { device: Device };
+        type LastSeenChanged = { device: Device,
+            reason: 'deviceAnnounce' | 'networkAddress' | 'deviceJoined' | 'messageEmitted' | 'messageNonEmitted'; };
        type DeviceNetworkAddressChanged = { device: Device };
        type DeviceAnnounce = { device: Device };
        type DeviceInterview = { device: Device, status: 'started' | 'successful' | 'failed' };
        type DeviceJoined = { device: Device };
-        type ReportingDisabled = { device: Device };
+        type EntityOptionsChanged = { entity: Device | Group, from: KeyValue, to: KeyValue };
+        type Reconfigure = { device: Device };
        type DeviceLeave = { ieeeAddr: string, name: string };
        type GroupMembersChanged = {group: Group, action: 'remove' | 'add' | 'remove_all',
            endpoint: zh.Endpoint, skipDisableReporting: boolean };
-        type PublishEntityState = {entity: Group | Device, message: KeyValue, stateChangeReason: StateChangeReason };
+        type PublishEntityState = {entity: Group | Device, message: KeyValue, stateChangeReason: StateChangeReason,
+                payload: KeyValue};
        type DeviceMessage = {
            type: ZHEvents.MessagePayloadType;
            device: Device;
@@ -161,24 +166,18 @@ declare global {
    // Settings
    // eslint-disable camelcase
    interface Settings {
-        homeassistant?: boolean,
-        devices?: {[s: string]: DeviceSettings},
-        groups?: {[s: string]: GroupSettings},
-        passlist: string[],
-        blocklist: string[],
-        whitelist: string[],
-        ban: string[],
-        availability?: boolean | {
-            active?: {timeout?: number},
-            passive?: {timeout?: number}
+        homeassistant?: {
+            discovery_topic: string,
+            status_topic: string,
+            legacy_entity_attributes: boolean,
+            legacy_triggers: boolean,
        },
-        permit_join: boolean,
-        frontend?: {
-            auth_token?: string,
-            host?: string,
-            port?: number,
-            url?: string,
+        permit_join?: boolean,
+        availability?: {
+            active: {timeout: number},
+            passive: {timeout: number}
        },
+        external_converters: string[],
        mqtt: {
            base_topic: string,
            include_device_information: boolean,
@@ -195,11 +194,14 @@ declare global {
            reject_unauthorized?: boolean,
        },
        serial: {
-            disable_led?: boolean,
+            disable_led: boolean,
            port?: string,
-            adapter?: 'deconz' | 'zstack' | 'ezsp' | 'zigate'
+            adapter?: 'deconz' | 'zstack' | 'ezsp' | 'zigate',
+            baudrate?: number,
+            rtscts?: boolean,
        },
-        device_options: KeyValue,
+        passlist: string[],
+        blocklist: string[],
        map_options: {
            graphviz: {
                colors: {
@@ -220,12 +222,24 @@ declare global {
                },
            },
        },
-        experimental: {
-            output: 'json' | 'attribute' | 'attribute_and_json',
-            transmit_power?: number,
+        ota: {
+            update_check_interval: number,
+            disable_automatic_update_check: boolean,
+            zigbee_ota_override_index_location?: string,
+            ikea_ota_use_test_url?: boolean,
        },
+        frontend?: {
+            auth_token?: string,
+            host?: string,
+            port?: number,
+            url?: string,
+        },
+        devices?: {[s: string]: DeviceOptions},
+        groups?: {[s: string]: GroupOptions},
+        device_options: KeyValue,
        advanced: {
            legacy_api: boolean,
+            legacy_availability_payload: boolean,
            log_rotation: boolean,
            log_symlink_current: boolean,
            log_output: ('console' | 'file' | 'syslog')[],
@@ -233,7 +247,6 @@ declare global {
            log_file: string,
            log_level: 'debug' | 'info' | 'error' | 'warn',
            log_syslog: KeyValue,
-            soft_reset_timeout: number,
            pan_id: number | 'GENERATE',
            ext_pan_id: number[],
            channel: number,
@@ -245,30 +258,21 @@ declare global {
            last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' | 'epoch',
            elapsed: boolean,
            network_key: number[] | 'GENERATE',
-            report: boolean,
-            homeassistant_discovery_topic: string,
-            homeassistant_status_topic: string,
-            homeassistant_legacy_entity_attributes: boolean,
-            homeassistant_legacy_triggers: boolean,
            timestamp_format: string,
-            baudrate?: number,
-            rtscts?: boolean,
-            ikea_ota_use_test_url?: boolean,
-            // below are deprecated
+            output: 'json' | 'attribute' | 'attribute_and_json',
+            transmit_power?: number,
+            // Everything below is deprecated
            availability_timeout?: number,
            availability_blocklist?: string[],
            availability_passlist?: string[],
            availability_blacklist?: string[],
            availability_whitelist?: string[],
+            soft_reset_timeout: number,
+            report: boolean,
        },
-        ota: {
-            update_check_interval: number,
-            disable_automatic_update_check: boolean
-        },
-        external_converters: string[],
    }

-    interface DeviceSettings {
+    interface DeviceOptions {
        ID?: string,
        retention?: number,
        availability?: boolean | {timeout: number},
@@ -276,23 +280,30 @@ declare global {
        retrieve_state?: boolean,
        debounce?: number,
        debounce_ignore?: string[],
+        filtered_attributes?: string[],
+        filtered_cache?: string[],
        filtered_optimistic?: string[],
        icon?: string,
        homeassistant?: KeyValue,
        legacy?: boolean,
-        filtered_attributes?: string[],
        friendly_name: string,
+        description?: string,
+        qos?: 0 | 1 | 2,
    }

-    interface GroupSettings {
+    interface GroupOptions {
        devices?: string[],
        ID?: number,
        optimistic?: boolean,
+        off_state?: 'all_members_off' | 'last_member_state'
+        filtered_attributes?: string[],
+        filtered_cache?: string[],
        filtered_optimistic?: string[],
        retrieve_state?: boolean,
        homeassistant?: KeyValue,
-        filtered_attributes?: string[],
        friendly_name: string,
+        description?: string,
+        qos?: 0 | 1 | 2,
    }
 }

@@ -4,10 +4,15 @@ declare module 'zigbee-herdsman-converters' {
    export const toZigbeeConverters: {[s: string]: zhc.ToZigbeeConverter};
    export const definitions: zhc.Definition[];
    export function addDeviceDefinition(definition: zhc.Definition): Promise<void>;
-    export function onEvent(type: string, data: KeyValue, device: zh.Device): Promise<void>;
+    export function onEvent(type: string, data: KeyValue, device: zh.Device, settings: KeyValue,
+        state: KeyValue): Promise<void>;
 }

 declare module 'zigbee-herdsman-converters/lib/ota/tradfri' {
    export function useTestURL(): void;
 }

+declare module 'zigbee-herdsman-converters/lib/ota/zigbeeOTA' {
+    export function useIndexOverride(indexFileName: string): void;
+    export function setDataDir(dataDir: string): void;
+}
@@ -0,0 +1,9 @@
+declare module 'zigbee2mqtt-frontend' {
+    export function getPath(): string;
+}
+
+declare module 'connect-gzip-static' {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    export type RequestHandler = (req: any, res: any) => void;
+    export default function gzipStatic(root: string, options?: Record<string, unknown>): RequestHandler;
+}
@@ -1,3 +0,0 @@
-declare module 'zigbee2mqtt-frontend' {
-    export function getPath(): string;
-}
@@ -14,7 +14,7 @@ function load(): void {
 load();

 function joinPath(file: string): string {
-    return path.join(dataPath, file);
+    return path.resolve(dataPath, file);
 }

 function getPath(): string {
@@ -22,8 +22,8 @@ function getPath(): string {
 }

 // eslint-disable-next-line camelcase
-function __testingOnly_reload(): void {
+function testingOnlyReload(): void {
    load();
 }

-export default {joinPath, getPath, __testingOnly_reload};
+export default {joinPath, getPath, testingOnlyReload};
@@ -9,40 +9,12 @@ import assert from 'assert';

 const colorizer = winston.format.colorize();

-// What transports to enable
-const output = settings.get().advanced.log_output;
-
-// Directory to log to
-const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
-const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
-const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
-
-// Make sure that log directoy exsists when not logging to stdout only
-if (output.includes('file')) {
-    fx.mkdirSync(directory);
-
-    if (settings.get().advanced.log_symlink_current) {
-        const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
-        const actual = './' + timestamp;
-        if (fs.existsSync(current)) {
-            fs.unlinkSync(current);
-        }
-        fs.symlinkSync(actual, current);
-    }
-}
-
 type Z2MLogLevel = 'warn' | 'debug' | 'info' | 'error';
 type WinstonLogLevel = 'warning' | 'debug' | 'info' | 'error';

 const z2mToWinstonLevel = (level: Z2MLogLevel): WinstonLogLevel => level === 'warn' ? 'warning' : level;
 const winstonToZ2mLevel = (level: WinstonLogLevel): Z2MLogLevel => level === 'warning' ? 'warn' : level;

-// Determine the log level.
-const z2mLevel = settings.get().advanced.log_level;
-const validLevels = ['info', 'error', 'warn', 'debug'];
-assert(validLevels.includes(z2mLevel), `'${z2mLevel}' is not valid log_level, use one of '${validLevels.join(', ')}'`);
-const level = z2mToWinstonLevel(z2mLevel);
-
 const levelWithCompensatedLength: {[s: string]: string} = {
    'info': 'info ',
    'error': 'error',
@@ -50,69 +22,113 @@ const levelWithCompensatedLength: {[s: string]: string} = {
    'debug': 'debug',
 };

-const timestampFormat = (): string => moment().format(settings.get().advanced.timestamp_format);
+let logger: winston.Logger;
+let fileTransport : winston.transport;
+let output: string[];
+let directory: string;
+let logFilename: string;
+let transportsToUse: winston.transport[];

-// Setup default console logger
-const transportsToUse: winston.transport[] = [
-    new winston.transports.Console({
+function init(): void {
+    // What transports to enable
+    output = settings.get().advanced.log_output;
+
+    // Directory to log to
+    const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
+    directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+    logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
+
+    // Make sure that log directoy exsists when not logging to stdout only
+    if (output.includes('file')) {
+        fx.mkdirSync(directory);
+
+        if (settings.get().advanced.log_symlink_current) {
+            const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
+            const actual = './' + timestamp;
+            /* istanbul ignore next */
+            if (fs.existsSync(current)) {
+                fs.unlinkSync(current);
+            }
+            fs.symlinkSync(actual, current);
+        }
+    }
+
+    // Determine the log level.
+    const z2mLevel = settings.get().advanced.log_level;
+    const validLevels = ['info', 'error', 'warn', 'debug'];
+    assert(validLevels.includes(z2mLevel),
+        `'${z2mLevel}' is not valid log_level, use one of '${validLevels.join(', ')}'`);
+    const level = z2mToWinstonLevel(z2mLevel);
+
+    const timestampFormat = (): string => moment().format(settings.get().advanced.timestamp_format);
+
+    // Setup default console logger
+    transportsToUse = [
+        new winston.transports.Console({
+            level,
+            silent: !output.includes('console'),
+            format: winston.format.combine(
+                winston.format.timestamp({format: timestampFormat}),
+                winston.format.printf(/* istanbul ignore next */(info) => {
+                    const {timestamp, level, message} = info;
+                    const l = winstonToZ2mLevel(level as WinstonLogLevel);
+
+                    const plainPrefix = `Zigbee2MQTT:${levelWithCompensatedLength[l]}`;
+                    let prefix = plainPrefix;
+                    if (process.stdout.isTTY) {
+                        prefix = colorizer.colorize(l, plainPrefix);
+                    }
+                    return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+                }),
+            ),
+        }),
+    ];
+
+    // Add file logger when enabled
+    // NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+    const transportFileOptions: KeyValue = {
+        filename: path.join(directory, logFilename),
+        json: false,
        level,
-        silent: !output.includes('console'),
        format: winston.format.combine(
            winston.format.timestamp({format: timestampFormat}),
            winston.format.printf(/* istanbul ignore next */(info) => {
                const {timestamp, level, message} = info;
                const l = winstonToZ2mLevel(level as WinstonLogLevel);
-                const prefix = colorizer.colorize(l, `Zigbee2MQTT:${levelWithCompensatedLength[l]}`);
-                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+                return `${levelWithCompensatedLength[l]} ${timestamp.split('.')[0]}: ${message}`;
            }),
        ),
-    }),
-];
-
-// Add file logger when enabled
-// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
-const transportFileOptions: KeyValue = {
-    filename: path.join(directory, logFilename),
-    json: false,
-    level,
-    format: winston.format.combine(
-        winston.format.timestamp({format: timestampFormat}),
-        winston.format.printf(/* istanbul ignore next */(info) => {
-            const {timestamp, level, message} = info;
-            const l = winstonToZ2mLevel(level as WinstonLogLevel);
-            return `${levelWithCompensatedLength[l]} ${timestamp.split('.')[0]}: ${message}`;
-        }),
-    ),
-};
-
-if (settings.get().advanced.log_rotation) {
-    transportFileOptions.tailable = true;
-    transportFileOptions.maxFiles = 3; // Keep last 3 files
-    transportFileOptions.maxsize = 10000000; // 10MB
-}
-
-if (output.includes('file')) {
-    transportsToUse.push(new winston.transports.File(transportFileOptions));
-}
-
-/* istanbul ignore next */
-if (output.includes('syslog')) {
-    // eslint-disable-next-line
-    require('winston-syslog').Syslog;
-    const options: KeyValue = {
-        app_name: 'Zigbee2MQTT',
-        format: winston.format.printf(/* istanbul ignore next */(info) => {
-            return `${info.message}`;
-        }),
-        ...settings.get().advanced.log_syslog,
    };
-    if (options.hasOwnProperty('type')) options.type = options.type.toString();
-    // @ts-ignore
-    transportsToUse.push(new winston.transports.Syslog(options));
-}

-// Create logger
-const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
+    if (settings.get().advanced.log_rotation) {
+        transportFileOptions.tailable = true;
+        transportFileOptions.maxFiles = 3; // Keep last 3 files
+        transportFileOptions.maxsize = 10000000; // 10MB
+    }
+
+    if (output.includes('file')) {
+        fileTransport = new winston.transports.File(transportFileOptions);
+        transportsToUse.push(fileTransport);
+    }
+
+    /* istanbul ignore next */
+    if (output.includes('syslog')) {
+        // eslint-disable-next-line
+        require('winston-syslog').Syslog;
+        const options: KeyValue = {
+            app_name: 'Zigbee2MQTT',
+            format: winston.format.printf(/* istanbul ignore next */(info) => {
+                return `${info.message}`;
+            }),
+            ...settings.get().advanced.log_syslog,
+        };
+        if (options.hasOwnProperty('type')) options.type = options.type.toString();
+        // @ts-ignore
+        transportsToUse.push(new winston.transports.Syslog(options));
+    }
+
+    logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
+}

 // Cleanup any old log directory.
 function cleanup(): void {
@@ -181,6 +197,29 @@ function error(message: string): void {
    logger.error(message);
 }

+// Workaround for https://github.com/winstonjs/winston/issues/1629.
+// https://github.com/Koenkk/zigbee2mqtt/pull/10905
+/* istanbul ignore next */
+async function end(): Promise<void> {
+    logger.end();
+
+    await new Promise<void>((resolve) => {
+        if (!fileTransport) {
+            process.nextTick(resolve);
+        } else {
+            // @ts-ignore
+            if (fileTransport._dest) {
+                // @ts-ignore
+                fileTransport._dest.on('finish', resolve);
+            } else {
+                // @ts-ignore
+                fileTransport.on('open', () => fileTransport._dest.on('finish', resolve));
+            }
+        }
+    });
+}
+
 export default {
-    logOutput, warn, warning, error, info, debug, setLevel, getLevel, cleanup, addTransport, winston: logger,
+    init, logOutput, warn, warning, error, info, debug, setLevel, getLevel, cleanup, addTransport, end,
+    winston: (): winston.Logger => logger,
 };
@@ -1,14 +1,49 @@
 {
  "type": "object",
  "properties": {
-    "device_options": {
-      "type": "object"
-    },
    "homeassistant": {
      "title": "Home Assistant integration",
-      "type": "boolean",
+      "requiresRestart": true,
      "description": "Home Assistant integration (MQTT discovery)",
-      "default": false
+      "default": false,
+      "oneOf": [
+        {
+          "type": "boolean",
+          "title": "Home Assistant (simple)"
+        },
+        {
+          "type": "object",
+          "title": "Home Assistant (advanced)",
+          "properties": {
+            "legacy_triggers": {
+              "type": "boolean",
+              "title": "Home Assistant legacy triggers",
+              "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
+              "default": true
+            },
+            "discovery_topic": {
+              "type": "string",
+              "title": "Homeassistant discovery topic",
+              "description": "Home Assistant discovery topic",
+              "requiresRestart": true,
+              "examples": ["homeassistant"]
+            },
+            "legacy_entity_attributes": {
+              "type": "boolean",
+              "title": "Home Assistant legacy entity attributes",
+              "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+              "default": true
+            },
+            "status_topic": {
+              "type": "string",
+              "title": "Home Assistant status topic",
+              "description": "Home Assistant status topic",
+              "requiresRestart": true,
+              "examples": ["homeassistant/status"]
+            }
+          }
+        }
+      ]
    },
    "permit_join": {
      "type": "boolean",
@@ -16,25 +51,15 @@
      "title": "Permit join",
      "description": "Allow new devices to join (re-applied at restart)"
    },
-    "external_converters": {
-      "type": "array",
-      "title": "External converters",
-      "description": "You can define external converters to e.g. add support for a DiY device",
-      "requiresRestart": true,
-      "items": {
-        "type": "string"
-      },
-      "examples": ["DIYRuZ_FreePad.js"]
-    },
    "availability": {
      "oneOf": [
        {
          "type": "boolean",
-          "title": "Availability (boolean)"
+          "title": "Availability (simple)"
        },
        {
          "type": "object",
-          "title": "Availability (object)",
+          "title": "Availability (advanced)",
          "properties": {
            "active": {
              "type": "object",
@@ -53,9 +78,9 @@
            },
            "passive": {
              "type": "object",
-              "title": "Active",
+              "title": "Passive",
              "requiresRestart": true,
-              "description": "Options for passive devices (routers/mains powered)",
+              "description": "Options for passive devices (mostly battery powered)",
              "properties": {
                "timeout": {
                  "type": "number",
@@ -71,7 +96,17 @@
      ],
      "title": "Availability",
      "requiresRestart": true,
-      "description": "Checks wether devices are online/offline"
+      "description": "Checks whether devices are online/offline"
+    },
+    "external_converters": {
+      "type": "array",
+      "title": "External converters",
+      "description": "You can define external converters to e.g. add support for a DiY device",
+      "requiresRestart": true,
+      "items": {
+        "type": "string"
+      },
+      "examples": ["DIYRuZ_FreePad.js"]
    },
    "mqtt": {
      "type": "object",
@@ -80,6 +115,7 @@
        "base_topic": {
          "type": "string",
          "title": "Base topic",
+          "default": "zigbee2mqtt",
          "requiresRestart": true,
          "description": "MQTT base topic for Zigbee2MQTT MQTT messages",
          "examples": ["zigbee2mqtt"]
@@ -169,7 +205,7 @@
          "default": false
        }
      },
-      "required": ["base_topic", "server"]
+      "required": ["server"]
    },
    "serial": {
      "type": "object",
@@ -196,6 +232,19 @@
          "default": "auto",
          "requiresRestart": true,
          "description": "Adapter type, not needed unless you are experiencing problems"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
        }
      }
    },
@@ -217,331 +266,6 @@
        "type": "string"
      }
    },
-    "whitelist": {
-      "readOnly": true,
-      "type": "array",
-      "requiresRestart": true,
-      "title": "Whitelist (deprecated, use passlist)",
-      "items": {
-        "type": "string"
-      }
-    },
-    "ban": {
-      "readOnly": true,
-      "type": "array",
-      "requiresRestart": true,
-      "title": "Ban (deprecated, use blocklist)",
-      "items": {
-        "type": "string"
-      }
-    },
-    "experimental": {
-      "type": "object",
-      "title": "Experimental",
-      "properties": {
-        "transmit_power": {
-          "type": ["number", "null"],
-          "title": "Transmit power",
-          "requiresRestart": true,
-          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
-        },
-        "output": {
-          "type": "string",
-          "enum": ["attribute_and_json", "attribute", "json"],
-          "title": "MQTT output type",
-          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
-        }
-      }
-    },
-    "advanced": {
-      "type": "object",
-      "title": "Advanced",
-      "properties": {
-        "legacy_api": {
-          "type": "boolean",
-          "title": "Legacy API",
-          "requiresRestart": true,
-          "description": "Disables the legacy api (false = disable)",
-          "default": true
-        },
-        "pan_id": {
-          "oneOf": [{
-              "type": "string",
-              "title": "Pan ID (string)"
-            },
-            {
-              "type": "number",
-              "title": "Pan ID (number)"
-            }
-          ],
-          "title": "Pan ID",
-          "requiresRestart": true,
-          "description": "ZigBee pan ID, changing requires repairing all devices!"
-        },
-        "ext_pan_id": {
-          "type": "array",
-          "items": {
-            "type": "number"
-          },
-          "title": "Ext Pan ID",
-          "requiresRestart": true,
-          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
-        },
-        "channel": {
-          "type": "number",
-          "minimum": 11,
-          "maximum": 26,
-          "default": 11,
-          "title": "ZigBee channel",
-          "requiresRestart": true,
-          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
-          "examples": [15, 20, 25]
-        },
-        "cache_state": {
-          "type": "boolean",
-          "title": "Cache state",
-          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
-          "default": true
-        },
-        "cache_state_persistent": {
-          "type": "boolean",
-          "title": "Persist cache state",
-          "description": "Persist cached state, only used when cache_state: true",
-          "default": true
-        },
-        "cache_state_send_on_startup": {
-          "type": "boolean",
-          "title": "Send cached state on startup",
-          "description": "Send cached state on startup, only used when cache_state: true",
-          "default": true
-        },
-        "log_rotation": {
-          "type": "boolean",
-          "title": "Log rotation",
-          "requiresRestart": true,
-          "description": "Log rotation",
-          "default": true
-        },
-        "log_symlink_current": {
-          "type": "boolean",
-          "title": "Log symlink current",
-          "requiresRestart": true,
-          "description": "Create symlink to current logs in the log directory",
-          "default": false
-        },
-        "log_level": {
-          "type": "string",
-          "enum": ["info", "warn", "error", "debug"],
-          "title": "Log level",
-          "description": "Logging level",
-          "default": "info"
-        },
-        "log_output": {
-          "type": "array",
-          "requiresRestart": true,
-          "items": {
-            "type": "string",
-            "enum": ["console", "file", "syslog"]
-          },
-          "title": "Log output",
-          "description": "Output location of the log, leave empty to supress logging"
-        },
-        "log_directory": {
-          "type": "string",
-          "title": "Log directory",
-          "requiresRestart": true,
-          "description": "Location of log directory",
-          "examples": ["data/log/%TIMESTAMP%"]
-        },
-        "log_file": {
-          "type": "string",
-          "title": "Log file",
-          "requiresRestart": true,
-          "description": "Log file name, can also contain timestamp",
-          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
-          "default": "log.txt"
-        },
-        "baudrate": {
-          "type": "number",
-          "title": "Baudrate",
-          "requiresRestart": true,
-          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
-          "examples": [38400, 57600, 115200]
-        },
-        "rtscts": {
-          "type": "boolean",
-          "title": "RTS / CTS",
-          "requiresRestart": true,
-          "description": "RTS / CTS Hardware Flow Control for serial port"
-        },
-        "soft_reset_timeout": {
-          "type": "number",
-          "minimum": 0,
-          "requiresRestart": true,
-          "title": "Soft reset timeout (deprecated)",
-          "description": "Soft reset ZNP after timeout",
-          "readOnly": true
-        },
-        "network_key": {
-          "oneOf": [{
-              "type": "string",
-              "title": "Network key(string)"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "number"
-              },
-              "title": "Network key(array)"
-            }
-          ],
-          "title": "Network key",
-          "requiresRestart": true,
-          "description": "Network encryption key, changing requires repairing all devices!"
-        },
-        "last_seen": {
-          "type": "string",
-          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
-          "title": "Last seen",
-          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
-          "default": "disable"
-        },
-        "elapsed": {
-          "type": "boolean",
-          "title": "Elapsed",
-          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
-          "default": false
-        },
-        "report": {
-          "type": "boolean",
-          "title": "Reporting",
-          "requiresRestart": true,
-          "readOnly": true,
-          "description": "Enables report feature (deprecated)"
-        },
-        "homeassistant_discovery_topic": {
-          "type": "string",
-          "title": "Homeassistant discovery topic",
-          "description": "Home Assistant discovery topic",
-          "requiresRestart": true,
-          "examples": ["homeassistant"]
-        },
-        "homeassistant_legacy_entity_attributes": {
-          "type": "boolean",
-          "title": "Home Assistant legacy entity attributes",
-          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
-          "default": true
-        },
-        "homeassistant_status_topic": {
-          "type": "string",
-          "title": "Home Assistant status topic",
-          "description": "Home Assistant status topic",
-          "requiresRestart": true,
-          "examples": ["homeassistant/status"]
-        },
-        "timestamp_format": {
-          "type": "string",
-          "title": "Timestamp format",
-          "requiresRestart": true,
-          "description": "Log timestamp format",
-          "examples": ["YYYY-MM-DD HH:mm:ss"]
-        },
-        "adapter_concurrent": {
-          "title": "Adapter concurrency",
-          "requiresRestart": true,
-          "type": ["number", "null"],
-          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
-        },
-        "adapter_delay": {
-          "type": ["number", "null"],
-          "requiresRestart": true,
-          "title": "Adapter delay",
-          "description": "Adapter delay"
-        },
-        "ikea_ota_use_test_url": {
-          "type": "boolean",
-          "title": "IKEA TRADFRI OTA use test url",
-          "requiresRestart": true,
-          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
-          "default": false
-        },
-        "homeassistant_legacy_triggers": {
-          "type": "boolean",
-          "title": "Home Assistant legacy triggers",
-          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
-          "default": true
-        },
-        "log_syslog": {
-          "type": "object",
-          "title": "syslog",
-          "properties": {
-            "host": {
-              "type": "string",
-              "title": "Host",
-              "description": "The host running syslogd, defaults to localhost.",
-              "default": "localhost"
-            },
-            "port": {
-              "type": "number",
-              "title": "Port",
-              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
-              "default": 123
-            },
-            "protocol": {
-              "type": "string",
-              "title": "Protocol",
-              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
-              "default": "tcp4",
-              "examples": [
-                "udp4",
-                "tls4",
-                "unix",
-                "unix-connect"
-              ]
-            },
-            "path": {
-              "type": "string",
-              "title": "Path",
-              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
-              "default": "/dev/log",
-              "examples": [
-                "/var/run/syslog"
-              ]
-            },
-            "pid": {
-              "type": "string",
-              "title": "PID",
-              "description": "PID of the process that log messages are coming from (Default process.pid).",
-              "default": "process.pid"
-            },
-            "localhost": {
-              "type": "string",
-              "title": "Localhost",
-              "description": "Host to indicate that log messages are coming from (Default: localhost).",
-              "default": "localhost"
-            },
-            "type": {
-              "type": "string",
-              "title": "Type",
-              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
-              "default": "5424"
-            },
-            "app_name": {
-              "type": "string",
-              "title": "Localhost",
-              "description": "The name of the application (Default: Zigbee2MQTT).",
-              "default": "Zigbee2MQTT"
-            },
-            "eol": {
-              "type": "string",
-              "title": "eol",
-              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
-              "default": "/n"
-            }
-          }
-        }
-      }
-    },
    "map_options": {
      "type": "object",
      "title": "Networkmap",
@@ -612,9 +336,65 @@
          "title": "Disable automatic update check",
          "description": "Zigbee devices may request a firmware update, and do so frequently, causing Zigbee2MQTT to reach out to third party servers. If you disable these device initiated checks, you can still initiate a firmware update check manually.",
          "default": false
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
+        },
+        "zigbee_ota_override_index_location": {
+          "type": "string",
+          "title": "OTA index override file name",
+          "requiresRestart": true,
+          "description": "Location of override OTA index file",
+          "examples": ["index.json"]
        }
      }
    },
+    "frontend": {
+      "oneOf": [
+        {
+          "type": "boolean",
+          "title": "Frontend (simple)"
+        },
+        {
+          "type": "object",
+          "title": "Frontend (advanced)",
+          "properties": {
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "Frontend binding port",
+              "default": 8080,
+              "requiresRestart": true
+            },
+            "host": {
+              "type": "string",
+              "title": "Bind host",
+              "description": "Frontend binding host",
+              "default": "0.0.0.0",
+              "requiresRestart": true
+            },
+            "auth_token": {
+              "type": ["string", "null"],
+              "title": "Auth token",
+              "description": "Enables authentication, disabled by default",
+              "requiresRestart": true
+            },
+            "url": {
+              "type": ["string", "null"],
+              "title": "URL",
+              "description": "URL on which the frontend can be reached, currently only used for the Home Assistant device configuration page",
+              "requiresRestart": true
+            }
+          }
+        }
+      ],
+      "title": "Frontend",
+      "requiresRestart": true
+    },
    "devices": {
      "type": "object",
      "propertyNames": {
@@ -637,31 +417,353 @@
        }
      }
    },
-    "frontend": {
+    "device_options": {
      "type": "object",
-      "title": "Frontend",
+      "title": "Options that are applied to all devices"
+    },
+    "advanced": {
+      "type": "object",
+      "title": "Advanced",
      "properties": {
-        "port": {
-          "type": "number",
-          "title": "Port",
-          "description": "Frontend binding port",
-          "default": 8080,
-          "requiresRestart": true
+        "legacy_api": {
+          "type": "boolean",
+          "title": "Legacy API",
+          "requiresRestart": true,
+          "description": "Disables the legacy api (false = disable)",
+          "default": true
        },
-        "host": {
+        "legacy_availability_payload": {
+          "type": "boolean",
+          "title": "Legacy availability payload",
+          "requiresRestart": true,
+          "description": "Payload to be used for device availabilty and bridge/state topics. true = text, false = JSON",
+          "default": true
+        },
+        "log_rotation": {
+          "type": "boolean",
+          "title": "Log rotation",
+          "requiresRestart": true,
+          "description": "Log rotation",
+          "default": true
+        },
+        "log_symlink_current": {
+          "type": "boolean",
+          "title": "Log symlink current",
+          "requiresRestart": true,
+          "description": "Create symlink to current logs in the log directory",
+          "default": false
+        },
+        "log_output": {
+          "type": "array",
+          "requiresRestart": true,
+          "items": {
+            "type": "string",
+            "enum": ["console", "file", "syslog"]
+          },
+          "title": "Log output",
+          "description": "Output location of the log, leave empty to suppress logging"
+        },
+        "log_directory": {
          "type": "string",
-          "title": "Bind host",
-          "description": "Frontend binding host",
-          "default":" 0.0.0.0",
-          "requiresRestart": true
+          "title": "Log directory",
+          "requiresRestart": true,
+          "description": "Location of log directory",
+          "examples": ["data/log/%TIMESTAMP%"]
        },
-        "auth_token": {
-          "type": ["string", "null"],
-          "title": "Auth token",
-          "description": "Enables authentication, disabled by default",
-          "requiresRestart": true
+        "log_file": {
+          "type": "string",
+          "title": "Log file",
+          "requiresRestart": true,
+          "description": "Log file name, can also contain timestamp",
+          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
+          "default": "log.txt"
+        },
+        "log_level": {
+          "type": "string",
+          "enum": ["info", "warn", "error", "debug"],
+          "title": "Log level",
+          "description": "Logging level",
+          "default": "info"
+        },
+        "log_syslog": {
+          "type": "object",
+          "title": "syslog",
+          "properties": {
+            "host": {
+              "type": "string",
+              "title": "Host",
+              "description": "The host running syslogd, defaults to localhost.",
+              "default": "localhost"
+            },
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
+              "default": 123
+            },
+            "protocol": {
+              "type": "string",
+              "title": "Protocol",
+              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
+              "default": "tcp4",
+              "examples": [
+                "udp4",
+                "tls4",
+                "unix",
+                "unix-connect"
+              ]
+            },
+            "path": {
+              "type": "string",
+              "title": "Path",
+              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
+              "default": "/dev/log",
+              "examples": [
+                "/var/run/syslog"
+              ]
+            },
+            "pid": {
+              "type": "string",
+              "title": "PID",
+              "description": "PID of the process that log messages are coming from (Default process.pid).",
+              "default": "process.pid"
+            },
+            "localhost": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "Host to indicate that log messages are coming from (Default: localhost).",
+              "default": "localhost"
+            },
+            "type": {
+              "type": "string",
+              "title": "Type",
+              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
+              "default": "5424"
+            },
+            "app_name": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "The name of the application (Default: Zigbee2MQTT).",
+              "default": "Zigbee2MQTT"
+            },
+            "eol": {
+              "type": "string",
+              "title": "eol",
+              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
+              "default": "/n"
+            }
+          }
+        },
+        "pan_id": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Pan ID (string)"
+            },
+            {
+              "type": "number",
+              "title": "Pan ID (number)"
+            }
+          ],
+          "title": "Pan ID",
+          "requiresRestart": true,
+          "description": "ZigBee pan ID, changing requires repairing all devices!"
+        },
+        "ext_pan_id": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "title": "Ext Pan ID",
+          "requiresRestart": true,
+          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
+        },
+        "channel": {
+          "type": "number",
+          "minimum": 11,
+          "maximum": 26,
+          "default": 11,
+          "title": "ZigBee channel",
+          "requiresRestart": true,
+          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
+          "examples": [15, 20, 25]
+        },
+        "adapter_concurrent": {
+          "title": "Adapter concurrency",
+          "requiresRestart": true,
+          "type": ["number", "null"],
+          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
+        },
+        "adapter_delay": {
+          "type": ["number", "null"],
+          "requiresRestart": true,
+          "title": "Adapter delay",
+          "description": "Adapter delay"
+        },
+        "cache_state": {
+          "type": "boolean",
+          "title": "Cache state",
+          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
+          "default": true
+        },
+        "cache_state_persistent": {
+          "type": "boolean",
+          "title": "Persist cache state",
+          "description": "Persist cached state, only used when cache_state: true",
+          "default": true
+        },
+        "cache_state_send_on_startup": {
+          "type": "boolean",
+          "title": "Send cached state on startup",
+          "description": "Send cached state on startup, only used when cache_state: true",
+          "default": true
+        },
+        "last_seen": {
+          "type": "string",
+          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
+          "title": "Last seen",
+          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
+          "default": "disable"
+        },
+        "elapsed": {
+          "type": "boolean",
+          "title": "Elapsed",
+          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
+          "default": false
+        },
+        "network_key": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Network key(string)"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "number"
+              },
+              "title": "Network key(array)"
+            }
+          ],
+          "title": "Network key",
+          "requiresRestart": true,
+          "description": "Network encryption key, changing requires repairing all devices!"
+        },
+        "timestamp_format": {
+          "type": "string",
+          "title": "Timestamp format",
+          "requiresRestart": true,
+          "description": "Log timestamp format",
+          "examples": ["YYYY-MM-DD HH:mm:ss"]
+        },
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        },
+        "homeassistant_discovery_topic": {
+          "type": "string",
+          "title": "Homeassistant discovery topic",
+          "description": "Home Assistant discovery topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant"]
+        },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
+        "homeassistant_status_topic": {
+          "type": "string",
+          "title": "Home Assistant status topic",
+          "description": "Home Assistant status topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant/status"]
+        },
+        "homeassistant_legacy_triggers": {
+          "type": "boolean",
+          "title": "Home Assistant legacy triggers",
+          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discovered",
+          "default": true
+        },
+        "soft_reset_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "requiresRestart": true,
+          "title": "Soft reset timeout (deprecated)",
+          "description": "Soft reset ZNP after timeout",
+          "readOnly": true
+        },
+        "report": {
+          "type": "boolean",
+          "title": "Reporting",
+          "requiresRestart": true,
+          "readOnly": true,
+          "description": "Enables report feature (deprecated)"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate (deprecated)",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS (deprecated)",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url (deprecated)",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
        }
      }
+    },
+    "experimental": {
+      "type": "object",
+      "title": "Experimental (deprecated)",
+      "properties": {
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        }
+      }
+    },
+    "whitelist": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Whitelist (deprecated, use passlist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "ban": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Ban (deprecated, use blocklist)",
+      "items": {
+        "type": "string"
+      }
    }
  },
  "required": ["mqtt"],
@@ -710,28 +812,48 @@
          "description": "Publish optimistic state after set",
          "default": true
        },
-        "filtered_optimistic": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "examples": ["color_mode", "color_temp", "color"],
-          "title": "Filtered optimistic attributes",
-          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
-        },
        "filtered_attributes": {
          "type": "array",
          "items": {
            "type": "string"
          },
-          "examples": ["temperature", "battery", "action"],
+          "examples": ["^temperature$", "^battery$", "^action$"],
          "title": "Filtered publish attributes",
-          "description": "Filter attributes from publish payload."
+          "description": "Filter attributes with regex from published payload."
+        },
+        "filtered_cache": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["^input_actions$"],
+          "title": "Filtered attributes from cache",
+          "description": "Filter attributes with regex from being added to the cache, this prevents the attribute from being in the published payload when the value didn't change."
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["^color_(mode|temp)$", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes with regex from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
        },
        "icon": {
          "type": "string",
          "title": "Icon",
-          "description": "The user-defined device icon for the frontend. It can be a link to an image (not a path to a file) or base64 encoded data URL like: image/svg+xml;base64,PHN2ZyB3aW....R0aD"
+          "description": "The user-defined device icon for the frontend. It can be a full URL link to an image (e.g. https://SOME.SITE/MODEL123.jpg) (you cannot use a path to a local file) or base64 encoded data URL (e.g. image/svg+xml;base64,PHN2ZyB3aW....R0aD)"
+        },
+        "homeassistant": {
+          "type": ["object", "null"],
+          "title": "Home Assistant",
+          "properties": {
+              "name": {
+                  "type": "string",
+                  "title": "Home Assistant name",
+                  "description": "Name of the device in Home Assistant"
+              }
+          }
        }
      },
      "required": ["friendly_name"]
@@ -757,6 +879,14 @@
        "qos": {
          "type": "number"
        },
+        "off_state": {
+          "type": ["string"],
+          "enum": ["all_members_off", "last_member_state"],
+          "title": "Group off state",
+          "default": "auto",
+          "requiresRestart": true,
+          "description": "Control when to publish state OFF for a group. 'all_members_off': only publish state OFF when all group memebers are in state OFF, 'last_member_state': publish state OFF whenever one of its members changes to OFF"
+        },
        "filtered_attributes": {
          "type": "array",
          "items": {
@@ -5,33 +5,46 @@ import path from 'path';
 import yaml from './yaml';
 import Ajv from 'ajv';
 import schemaJson from './settings.schema.json';
-export const schema = schemaJson;
+export let schema = schemaJson;
+// @ts-ignore
+schema = {};
+objectAssignDeep(schema, schemaJson);
+
+// Remove legacy settings from schema
+{
+    delete schema.properties.advanced.properties.homeassistant_discovery_topic;
+    delete schema.properties.advanced.properties.homeassistant_legacy_entity_attributes;
+    delete schema.properties.advanced.properties.homeassistant_legacy_triggers;
+    delete schema.properties.advanced.properties.homeassistant_status_topic;
+    delete schema.properties.advanced.properties.soft_reset_timeout;
+    delete schema.properties.advanced.properties.report;
+    delete schema.properties.advanced.properties.baudrate;
+    delete schema.properties.advanced.properties.rtscts;
+    delete schema.properties.advanced.properties.ikea_ota_use_test_url;
+    delete schema.properties.experimental;
+    delete schemaJson.properties.whitelist;
+    delete schemaJson.properties.ban;
+}

 // DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
 const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
-const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schemaJson);
 const ajvRestartRequired = new Ajv({allErrors: true})
-    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+    .addKeyword({keyword: 'requiresRestart', validate: (s: unknown) => !s}).compile(schemaJson);

 const defaults: RecursivePartial<Settings> = {
-    passlist: [],
-    blocklist: [],
-    // Deprecated: use block/passlist
-    whitelist: [],
-    ban: [],
    permit_join: false,
+    external_converters: [],
    mqtt: {
+        base_topic: 'zigbee2mqtt',
        include_device_information: false,
-        /**
-         * Configurable force disable retain flag on mqtt publish.
-         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
-         */
        force_disable_retain: false,
    },
    serial: {
        disable_led: false,
    },
-    device_options: {},
+    passlist: [],
+    blocklist: [],
    map_options: {
        graphviz: {
            colors: {
@@ -52,12 +65,14 @@ const defaults: RecursivePartial<Settings> = {
            },
        },
    },
-    experimental: {
-        // json or attribute or attribute_and_json
-        output: 'json',
+    ota: {
+        update_check_interval: 24 * 60,
+        disable_automatic_update_check: false,
    },
+    device_options: {},
    advanced: {
        legacy_api: true,
+        legacy_availability_payload: true,
        log_rotation: true,
        log_symlink_current: false,
        log_output: ['console', 'file'],
@@ -65,106 +80,127 @@ const defaults: RecursivePartial<Settings> = {
        log_file: 'log.txt',
        log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
        log_syslog: {},
-        soft_reset_timeout: 0,
        pan_id: 0x1a62,
        ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
        channel: 11,
        adapter_concurrent: null,
        adapter_delay: null,
-
-        // Availability timeout in seconds, disabled by default.
-        availability_blocklist: [],
-        availability_passlist: [],
-        // Deprecated, use block/passlist
-        availability_blacklist: [],
-        availability_whitelist: [],
-
-        /**
-         * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
-         * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
-         *
-         * Therefore Zigbee2MQTT BY DEFAULT caches all values and resend it with every message.
-         * advanced.cache_state in configuration.yaml allows to configure this.
-         * https://www.zigbee2mqtt.io/configuration/configuration.html
-         */
        cache_state: true,
        cache_state_persistent: true,
        cache_state_send_on_startup: true,
-
-        /**
-         * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
-         * "ISO_8601": ISO 8601 format
-         * "ISO_8601_local": Local ISO 8601 format (instead of UTC-based)
-         * "epoch": milliseconds elapsed since the UNIX epoch
-         * "disable": no last_seen attribute (default)
-         */
        last_seen: 'disable',
-
-        // Optional: Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg
        elapsed: false,
-
-        /**
-         * https://github.com/Koenkk/zigbee2mqtt/issues/685#issuecomment-449112250
-         *
-         * Network key will serve as the encryption key of your network.
-         * Changing this will require you to repair your devices.
-         */
        network_key: [1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13],
-
-        /**
-         * Enables reporting feature
-         */
-        report: false,
-
-        /**
-         * Home Assistant discovery topic
-         */
-        homeassistant_discovery_topic: 'homeassistant',
-
-        /**
-         * Home Assistant status topic
-         */
-        homeassistant_status_topic: 'hass/status',
-
-        /**
-         * Home Assistant legacy entity attributes, when enabled:
-         * Zigbee2MQTT will send additional states as attributes with each entity.
-         * For example, A temperature & humidity sensor will have 2 entities for
-         * the temperature and humidity, with this setting enabled both entities
-         * will also have an temperature and humidity attribute.
-         */
-        homeassistant_legacy_entity_attributes: true,
-
-        /**
-         * Home Assistant legacy triggers, when enabled:
-         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
-         * - A 'sensor_action' and 'sensor_click' will be discoverd
-         */
-        homeassistant_legacy_triggers: true,
-
-        /**
-         * Configurable timestampFormat
-         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
-         */
        timestamp_format: 'YYYY-MM-DD HH:mm:ss',
+        output: 'json',
+        // Everything below is deprecated
+        availability_blocklist: [],
+        availability_passlist: [],
+        availability_blacklist: [],
+        availability_whitelist: [],
+        soft_reset_timeout: 0,
+        report: false,
    },
-    ota: {
-        /**
-         * Minimal time delta in minutes between polling third party server for potential firmware updates
-         */
-        update_check_interval: 24 * 60,
-        /**
-         * Completely disallow Zigbee devices to initiate a search for a potential firmware update.
-         * If set to true, only a user-initiated update search will be possible.
-         */
-        disable_automatic_update_check: false,
-    },
-    external_converters: [],
 };

 let _settings: Partial<Settings>;
 let _settingsWithDefaults: Settings;

+function loadSettingsWithDefaults(): void {
+    _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
+
+    if (!_settingsWithDefaults.devices) {
+        _settingsWithDefaults.devices = {};
+    }
+
+    if (!_settingsWithDefaults.groups) {
+        _settingsWithDefaults.groups = {};
+    }
+
+    if (_settingsWithDefaults.homeassistant) {
+        const defaults = {discovery_topic: 'homeassistant', status_topic: 'hass/status',
+            legacy_entity_attributes: true, legacy_triggers: true};
+        const sLegacy = {};
+        if (_settingsWithDefaults.advanced) {
+            for (const key of ['homeassistant_legacy_triggers', 'homeassistant_discovery_topic',
+                'homeassistant_legacy_entity_attributes', 'homeassistant_status_topic']) {
+                // @ts-ignore
+                if (_settingsWithDefaults.advanced[key] !== undefined) {
+                    // @ts-ignore
+                    sLegacy[key.replace('homeassistant_', '')] = _settingsWithDefaults.advanced[key];
+                }
+            }
+        }
+
+        const s = typeof _settingsWithDefaults.homeassistant === 'object' ? _settingsWithDefaults.homeassistant : {};
+        // @ts-ignore
+        _settingsWithDefaults.homeassistant = {};
+        objectAssignDeep(_settingsWithDefaults.homeassistant, defaults, sLegacy, s);
+    }
+
+    if (_settingsWithDefaults.availability || _settingsWithDefaults.advanced?.availability_timeout) {
+        const defaults = {};
+        const s = typeof _settingsWithDefaults.availability === 'object' ? _settingsWithDefaults.availability : {};
+        // @ts-ignore
+        _settingsWithDefaults.availability = {};
+        objectAssignDeep(_settingsWithDefaults.availability, defaults, s);
+    }
+
+    if (_settingsWithDefaults.frontend) {
+        const defaults = {port: 8080, auth_token: false, host: '0.0.0.0'};
+        const s = typeof _settingsWithDefaults.frontend === 'object' ? _settingsWithDefaults.frontend : {};
+        // @ts-ignore
+        _settingsWithDefaults.frontend = {};
+        objectAssignDeep(_settingsWithDefaults.frontend, defaults, s);
+    }
+
+    if (_settings.advanced?.hasOwnProperty('baudrate') && _settings.serial?.baudrate == null) {
+        // @ts-ignore
+        _settingsWithDefaults.serial.baudrate = _settings.advanced.baudrate;
+    }
+
+    if (_settings.advanced?.hasOwnProperty('rtscts') && _settings.serial?.rtscts == null) {
+        // @ts-ignore
+        _settingsWithDefaults.serial.rtscts = _settings.advanced.rtscts;
+    }
+
+    if (_settings.advanced?.hasOwnProperty('ikea_ota_use_test_url') && _settings.ota?.ikea_ota_use_test_url == null) {
+        // @ts-ignore
+        _settingsWithDefaults.ota.ikea_ota_use_test_url = _settings.advanced.ikea_ota_use_test_url;
+    }
+
+    // @ts-ignore
+    if (_settings.experimental?.hasOwnProperty('transmit_power') && _settings.advanced?.transmit_power == null) {
+        // @ts-ignore
+        _settingsWithDefaults.advanced.transmit_power = _settings.experimental.transmit_power;
+    }
+
+    // @ts-ignore
+    if (_settings.experimental?.hasOwnProperty('output') && _settings.advanced?.output == null) {
+        // @ts-ignore
+        _settingsWithDefaults.advanced.output = _settings.experimental.output;
+    }
+
+    // @ts-ignore
+    _settingsWithDefaults.ban && _settingsWithDefaults.blocklist.push(..._settingsWithDefaults.ban);
+    // @ts-ignore
+    _settingsWithDefaults.whitelist && _settingsWithDefaults.passlist.push(..._settingsWithDefaults.whitelist);
+}
+
+function parseValueRef(text: string): {filename: string, key: string} | null {
+    const match = /!(.*) (.*)/g.exec(text);
+    if (match) {
+        let filename = match[1];
+        // This is mainly for backward compatibility.
+        if (!filename.endsWith('.yaml') && !filename.endsWith('.yml')) {
+            filename += '.yaml';
+        }
+        return {filename, key: match[2]};
+    } else {
+        return null;
+    }
+}
+
 function write(): void {
    const settings = getInternalSettings();
    const toWrite: KeyValue = objectAssignDeep({}, settings);
@@ -174,15 +210,16 @@ function write(): void {

    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
    for (const path of [
+        ['mqtt', 'server'],
        ['mqtt', 'user'],
        ['mqtt', 'password'],
        ['advanced', 'network_key'],
        ['frontend', 'auth_token'],
    ]) {
        if (actual[path[0]] && actual[path[0]][path[1]]) {
-            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
-            if (match) {
-                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+            const ref = parseValueRef(actual[path[0]][path[1]]);
+            if (ref) {
+                yaml.updateIfChanged(data.joinPath(ref.filename), ref.key, toWrite[path[0]][path[1]]);
                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
            }
        }
@@ -213,7 +250,7 @@ function write(): void {
    yaml.writeIfChanged(file, toWrite);

    _settings = read();
-    _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
+    loadSettingsWithDefaults();
 }

 export function validate(): string[] {
@@ -247,15 +284,18 @@ export function validate(): string[] {

    // Verify that all friendly names are unique
    const names: string[] = [];
-    const check = (name: string): void => {
-        if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
-        errors.push(...utils.validateFriendlyName(name));
-        names.push(name);
+    const check = (e: DeviceOptions | GroupOptions): void => {
+        if (names.includes(e.friendly_name)) errors.push(`Duplicate friendly_name '${e.friendly_name}' found`);
+        errors.push(...utils.validateFriendlyName(e.friendly_name));
+        names.push(e.friendly_name);
+        if (e.qos != null && ![0, 1, 2].includes(e.qos)) {
+            errors.push(`QOS for '${e.friendly_name}' not valid, should be 0, 1 or 2 got ${e.qos}`);
+        }
    };

    const settingsWithDefaults = get();
-    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
-    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g));

    if (settingsWithDefaults.mqtt.version !== 5) {
        for (const device of Object.values(settingsWithDefaults.devices)) {
@@ -283,26 +323,31 @@ export function validate(): string[] {

 function read(): Settings {
    const s = yaml.read(file) as Settings;
+    applyEnvironmentVariables(s);

    // Read !secret MQTT username and password if set
    // eslint-disable-next-line
    const interpetValue = (value: any): any => {
-        const re = /!(.*) (.*)/g;
-        const match = re.exec(value);
-        if (match) {
-            const file = data.joinPath(`${match[1]}.yaml`);
-            const key = match[2];
-            return yaml.read(file)[key];
+        const ref = parseValueRef(value);
+        if (ref) {
+            return yaml.read(data.joinPath(ref.filename))[ref.key];
        } else {
            return value;
        }
    };

-    if (s.mqtt?.user && s.mqtt?.password) {
+    if (s.mqtt?.user) {
        s.mqtt.user = interpetValue(s.mqtt.user);
+    }
+
+    if (s.mqtt?.password) {
        s.mqtt.password = interpetValue(s.mqtt.password);
    }

+    if (s.mqtt?.server) {
+        s.mqtt.server = interpetValue(s.mqtt.server);
+    }
+
    if (s.advanced?.network_key) {
        s.advanced.network_key = interpetValue(s.advanced.network_key);
    }
@@ -348,7 +393,11 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
                        }, settings);

                        if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
-                            setting[key] = JSON.parse(process.env[envVariableName]);
+                            try {
+                                setting[key] = JSON.parse(process.env[envVariableName]);
+                            } catch (error) {
+                                setting[key] = process.env[envVariableName];
+                            }
                        } else if (type.indexOf('number') >= 0) {
                            /* eslint-disable-line */ // @ts-ignore
                            setting[key] = process.env[envVariableName] * 1;
@@ -365,7 +414,7 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {

                if (typeof obj[key] === 'object' && obj[key]) {
                    const newPath = [...path];
-                    if (key !== 'properties') {
+                    if (key !== 'properties' && key !== 'oneOf' && !Number.isInteger(Number(key))) {
                        newPath.push(key);
                    }
                    iterate(obj[key], newPath);
@@ -373,13 +422,12 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
            }
        });
    };
-    iterate(schema.properties, []);
+    iterate(schemaJson.properties, []);
 }

 function getInternalSettings(): Partial<Settings> {
    if (!_settings) {
        _settings = read();
-        applyEnvironmentVariables(_settings);
    }

    return _settings;
@@ -387,15 +435,7 @@ function getInternalSettings(): Partial<Settings> {

 export function get(): Settings {
    if (!_settingsWithDefaults) {
-        _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
-    }
-
-    if (!_settingsWithDefaults.devices) {
-        _settingsWithDefaults.devices = {};
-    }
-
-    if (!_settingsWithDefaults.groups) {
-        _settingsWithDefaults.groups = {};
+        loadSettingsWithDefaults();
    }

    return _settingsWithDefaults;
@@ -440,7 +480,7 @@ export function apply(newSettings: Record<string, unknown>): boolean {
    return restartRequired;
 }

-export function getGroup(IDorName: string | number): GroupSettings {
+export function getGroup(IDorName: string | number): GroupOptions {
    const settings = get();
    const byID = settings.groups[IDorName];
    if (byID) {
@@ -456,14 +496,14 @@ export function getGroup(IDorName: string | number): GroupSettings {
    return null;
 }

-export function getGroups(): GroupSettings[] {
+export function getGroups(): GroupOptions[] {
    const settings = get();
    return Object.entries(settings.groups).map(([ID, group]) => {
        return {devices: [], ...group, ID: Number(ID)};
    });
 }

-function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
+function getGroupThrowIfNotExists(IDorName: string): GroupOptions {
    const group = getGroup(IDorName);
    if (!group) {
        throw new Error(`Group '${IDorName}' does not exist`);
@@ -472,7 +512,7 @@ function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
    return group;
 }

-export function getDevice(IDorName: string): DeviceSettings {
+export function getDevice(IDorName: string): DeviceOptions {
    const settings = get();
    const byID = settings.devices[IDorName];
    if (byID) {
@@ -488,7 +528,7 @@ export function getDevice(IDorName: string): DeviceSettings {
    return null;
 }

-function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
+function getDeviceThrowIfNotExists(IDorName: string): DeviceOptions {
    const device = getDevice(IDorName);
    if (!device) {
        throw new Error(`Device '${IDorName}' does not exist`);
@@ -497,7 +537,7 @@ function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
    return device;
 }

-export function addDevice(ID: string): DeviceSettings {
+export function addDevice(ID: string): DeviceOptions {
    if (getDevice(ID)) {
        throw new Error(`Device '${ID}' already exists`);
    }
@@ -513,17 +553,17 @@ export function addDevice(ID: string): DeviceSettings {
    return getDevice(ID);
 }

-export function whitelistDevice(ID: string): void {
+export function addDeviceToPasslist(ID: string): void {
    const settings = getInternalSettings();
-    if (!settings.whitelist) {
-        settings.whitelist = [];
+    if (!settings.passlist) {
+        settings.passlist = [];
    }

-    if (settings.whitelist.includes(ID)) {
-        throw new Error(`Device '${ID}' already whitelisted`);
+    if (settings.passlist.includes(ID)) {
+        throw new Error(`Device '${ID}' already in passlist`);
    }

-    settings.whitelist.push(ID);
+    settings.passlist.push(ID);
    write();
 }

@@ -537,16 +577,6 @@ export function blockDevice(ID: string): void {
    write();
 }

-export function banDevice(ID: string): void {
-    const settings = getInternalSettings();
-    if (!settings.ban) {
-        settings.ban = [];
-    }
-
-    settings.ban.push(ID);
-    write();
-}
-
 export function removeDevice(IDorName: string): void {
    const device = getDeviceThrowIfNotExists(IDorName);
    const settings = getInternalSettings();
@@ -564,7 +594,7 @@ export function removeDevice(IDorName: string): void {
    write();
 }

-export function addGroup(name: string, ID?: string): GroupSettings {
+export function addGroup(name: string, ID?: string): GroupOptions {
    utils.validateFriendlyName(name, true);
    if (getGroup(name) || getDevice(name)) {
        throw new Error(`friendly_name '${name}' is already in use`);
@@ -38,7 +38,7 @@ const endpointNames = [
    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
-    'heat', 'cool', 'water', 'meter', 'wifi',
+    'heat', 'cool', 'water', 'meter', 'wifi', 'no_occupancy_since',
 ];

 function capitalize(s: string): string {
@@ -70,13 +70,14 @@ async function getZigbee2MQTTVersion(includeCommitHash=true): Promise<{commitHas
                commitHash = commit.shortHash;
            }

+            commitHash = commitHash.trim();
            resolve({commitHash, version});
        });
    });
 }

 async function getDependencyVersion(depend: string): Promise<{version: string}> {
-    const packageJSON = await import(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
+    const packageJSON = await import(path.join(require.resolve(depend), '..', '..', 'package.json'));
    const version = packageJSON.version;
    return {version};
 }
@@ -209,6 +210,43 @@ function toSnakeCase(value: string | KeyValue): any {
    }
 }

+function charRange(start: string, stop: string): number[] {
+    const result = [];
+    for (let idx=start.charCodeAt(0), end=stop.charCodeAt(0); idx <=end; ++idx) {
+        result.push(idx);
+    }
+    return result;
+}
+
+const controlCharacters = [
+    ...charRange('\u0000', '\u001F'),
+    ...charRange('\u007f', '\u009F'),
+    ...charRange('\ufdd0', '\ufdef'),
+];
+
+function containsControlCharacter(str: string): boolean {
+    for (let i = 0; i < str.length; i++) {
+        const ch = str.charCodeAt(i);
+        if (controlCharacters.includes(ch) || [0xFFFE, 0xFFFF].includes(ch & 0xFFFF)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+function getAllFiles(path_: string): string[] {
+    const result = [];
+    for (let item of fs.readdirSync(path_)) {
+        item = path.join(path_, item);
+        if (fs.lstatSync(item).isFile()) {
+            result.push(item);
+        } else {
+            result.push(...getAllFiles(item));
+        }
+    }
+    return result;
+}
+
 function validateFriendlyName(name: string, throwFirstError=false): string[] {
    const errors = [];
    for (const endpointName of endpointNames) {
@@ -219,7 +257,7 @@ function validateFriendlyName(name: string, throwFirstError=false): string[] {

    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
-    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
+    if (containsControlCharacter(name)) errors.push(`friendly_name is not allowed to contain control char`);
    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
    if (name.includes('#') || name.includes('+')) {
@@ -244,9 +282,13 @@ function sanitizeImageParameter(parameter: string): string {
    return sanitized;
 }

-function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boolean {
-    if (device.settings.hasOwnProperty('availability')) {
-        return !!device.settings.availability;
+function isAvailabilityEnabledForEntity(entity: Device | Group, settings: Settings): boolean {
+    if (entity.isGroup()) {
+        return !entity.membersDevices().map((d) => isAvailabilityEnabledForEntity(d, settings)).includes(false);
+    }
+
+    if (entity.options.hasOwnProperty('availability')) {
+        return !!entity.options.availability;
    }

    // availability_timeout = deprecated
@@ -255,11 +297,11 @@ function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boo

    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
    if (passlist.length > 0) {
-        return passlist.includes(device.name) || passlist.includes(device.ieeeAddr);
+        return passlist.includes(entity.name) || passlist.includes(entity.ieeeAddr);
    }

    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
-    return !blocklist.includes(device.name) && !blocklist.includes(device.ieeeAddr);
+    return !blocklist.includes(entity.name) && !blocklist.includes(entity.ieeeAddr);
 }

 const entityIDRegex = new RegExp(`^(.+?)(?:/(${endpointNames.join('|')}|\\d+))?$`);
@@ -272,19 +314,60 @@ function isEndpoint(obj: unknown): obj is zh.Endpoint {
    return obj.constructor.name.toLowerCase() === 'endpoint';
 }

+function flatten<Type>(arr: Type[][]): Type[] {
+    return [].concat(...arr);
+}
+
+function arrayUnique<Type>(arr: Type[]): Type[] {
+    return [...new Set(arr)];
+}
+
 function isZHGroup(obj: unknown): obj is zh.Group {
    return obj.constructor.name.toLowerCase() === 'group';
 }

+function availabilityPayload(state: 'online' | 'offline', settings: Settings): string {
+    return settings.advanced.legacy_availability_payload ? state : JSON.stringify({state});
+}
+
 const hours = (hours: number): number => 1000 * 60 * 60 * hours;
 const minutes = (minutes: number): number => 1000 * 60 * minutes;
 const seconds = (seconds: number): number => 1000 * seconds;

+function publishLastSeen(data: eventdata.LastSeenChanged, settings: Settings, allowMessageEmitted: boolean,
+    publishEntityState: PublishEntityState): void {
+    /**
+     * Prevent 2 MQTT publishes when 1 message event is received;
+     * - In case reason == messageEmitted, receive.ts will only call this when it did not publish a
+     *      message based on the received zigbee message. In this case allowMessageEmitted has to be true.
+     * - In case reason !== messageEmitted, controller.ts will call this based on the zigbee-herdsman
+     *      lastSeenChanged event.
+     */
+    const allow = data.reason !== 'messageEmitted' || (data.reason === 'messageEmitted' && allowMessageEmitted);
+    if (settings.advanced.last_seen && settings.advanced.last_seen !== 'disable' && allow) {
+        publishEntityState(data.device, {}, 'lastSeenChanged');
+    }
+}
+
+function filterProperties(filter: string[], data: KeyValue): void {
+    if (filter) {
+        for (const property of Object.keys(data)) {
+            if (filter.find((p) => property.match(`^${p}$`))) {
+                delete data[property];
+            }
+        }
+    }
+}
+
+function clone(obj: KeyValue): KeyValue {
+    return JSON.parse(JSON.stringify(obj));
+}

 export default {
    endpointNames, capitalize, getZigbee2MQTTVersion, getDependencyVersion, formatDate, objectHasProperties,
    equalsPartial, getObjectProperty, getResponse, parseJSON, loadModuleFromText, loadModuleFromFile,
    getExternalConvertersDefinitions, removeNullPropertiesFromObject, toNetworkAddressHex, toSnakeCase,
    parseEntityID, isEndpoint, isZHGroup, hours, minutes, seconds, validateFriendlyName, sleep,
-    sanitizeImageParameter, isAvailabilityEnabledForDevice,
+    sanitizeImageParameter, isAvailabilityEnabledForEntity, publishLastSeen, availabilityPayload,
+    getAllFiles, filterProperties, flatten, arrayUnique, clone,
 };
@@ -36,8 +36,8 @@ export default class Zigbee {
            databaseBackupPath: data.joinPath('database.db.backup'),
            backupPath: data.joinPath('coordinator_backup.json'),
            serialPort: {
-                baudRate: settings.get().advanced.baudrate,
-                rtscts: settings.get().advanced.rtscts,
+                baudRate: settings.get().serial.baudrate,
+                rtscts: settings.get().serial.rtscts,
                path: settings.get().serial.port,
                adapter: settings.get().serial.adapter,
            },
@@ -63,7 +63,7 @@ export default class Zigbee {

        this.herdsman.on('adapterDisconnected', () => this.eventBus.emitAdapterDisconnected());
        this.herdsman.on('lastSeenChanged', (data: ZHEvents.LastSeenChangedPayload) => {
-            this.eventBus.emitLastSeenChanged({device: this.resolveDevice(data.device.ieeeAddr)});
+            this.eventBus.emitLastSeenChanged({device: this.resolveDevice(data.device.ieeeAddr), reason: data.reason});
        });
        this.herdsman.on('permitJoinChanged', (data: ZHEvents.PermitJoinChangedPayload) => {
            this.eventBus.emitPermitJoinChanged(data);
@@ -80,12 +80,14 @@ export default class Zigbee {
        });
        this.herdsman.on('deviceInterview', (data: ZHEvents.DeviceInterviewPayload) => {
            const device = this.resolveDevice(data.device.ieeeAddr);
+            /* istanbul ignore if */ if (!device) return; // Prevent potential race
            const d = {device, status: data.status};
            this.logDeviceInterview(d);
            this.eventBus.emitDeviceInterview(d);
        });
        this.herdsman.on('deviceJoined', (data: ZHEvents.DeviceJoinedPayload) => {
            const device = this.resolveDevice(data.device.ieeeAddr);
+            /* istanbul ignore if */ if (!device) return; // Prevent potential race
            logger.info(`Device '${device.name}' joined`);
            this.eventBus.emitDeviceJoined({device});
        });
@@ -98,7 +100,9 @@ export default class Zigbee {
            const device = this.resolveDevice(data.device.ieeeAddr);
            logger.debug(`Received Zigbee message from '${device.name}', type '${data.type}', ` +
                `cluster '${data.cluster}', data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
-                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``));
+                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``) +
+                (device.zh.type === 'Coordinator' ? `, ignoring since it is from coordinator` : ``));
+            if (device.zh.type === 'Coordinator') return;
            this.eventBus.emitDeviceMessage({...data, device});
        });

@@ -108,8 +112,8 @@ export default class Zigbee {

        for (const device of this.devices(false)) {
            // If a passlist is used, all other device will be removed from the network.
-            const passlist = settings.get().passlist.concat(settings.get().whitelist);
-            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            const passlist = settings.get().passlist;
+            const blocklist = settings.get().blocklist;
            const remove = async (device: Device): Promise<void> => {
                try {
                    await device.zh.removeFromNetwork();
@@ -128,14 +132,9 @@ export default class Zigbee {
            }
        }

-        // Check if we have to turn off the led
-        if (settings.get().serial.disable_led) {
-            await this.herdsman.setLED(false);
-        }
-
        // Check if we have to set a transmit power
-        if (settings.get().experimental.hasOwnProperty('transmit_power')) {
-            const transmitPower = settings.get().experimental.transmit_power;
+        if (settings.get().advanced.hasOwnProperty('transmit_power')) {
+            const transmitPower = settings.get().advanced.transmit_power;
            await this.herdsman.setTransmitPower(transmitPower);
            logger.info(`Set transmit power to '${transmitPower}'`);
        }
@@ -154,7 +153,8 @@ export default class Zigbee {
            } else {
                logger.warn(`Device '${name}' with Zigbee model '${data.device.zh.modelID}' and manufacturer name ` +
                    `'${data.device.zh.manufacturerName}' is NOT supported, ` +
-                    `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
+                    // eslint-disable-next-line max-len
+                    `please follow https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html`);
            }
        } else if (data.status === 'failed') {
            logger.error(`Failed to interview '${name}', device has not successfully been paired`);
@@ -183,6 +183,10 @@ export default class Zigbee {
        return this.herdsman.isStopping();
    }

+    async backup(): Promise<void> {
+        return this.herdsman.backup();
+    }
+
    async getNetworkParameters(): Promise<zh.NetworkParameters> {
        return this.herdsman.getNetworkParameters();
    }
@@ -219,7 +223,7 @@ export default class Zigbee {
        }
    }

-    private resolveDevice(ieeeAddr: string): Device {
+    @bind private resolveDevice(ieeeAddr: string): Device {
        if (!this.deviceLookup[ieeeAddr]) {
            const device = this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
            device && (this.deviceLookup[ieeeAddr] = new Device(device));
@@ -235,7 +239,7 @@ export default class Zigbee {
    private resolveGroup(groupID: number): Group {
        const group = this.herdsman.getGroupByID(Number(groupID));
        if (group && !this.groupLookup[groupID]) {
-            this.groupLookup[groupID] = new Group(group);
+            this.groupLookup[groupID] = new Group(group, this.resolveDevice);
        }

        return this.groupLookup[groupID];
@@ -275,8 +279,8 @@ export default class Zigbee {

    @bind private async acceptJoiningDeviceHandler(ieeeAddr: string): Promise<boolean> {
        // If passlist is set, all devices not on passlist will be rejected to join the network
-        const passlist = settings.get().passlist.concat(settings.get().whitelist);
-        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        const passlist = settings.get().passlist;
+        const blocklist = settings.get().blocklist;
        if (passlist.length > 0) {
            if (passlist.includes(ieeeAddr)) {
                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
@@ -306,6 +310,10 @@ export default class Zigbee {
        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
    }

+    async addInstallCode(installCode: string): Promise<void> {
+        await this.herdsman.addInstallCode(installCode);
+    }
+
    async touchlinkIdentify(ieeeAddr: string, channel: number): Promise<void> {
        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
    }
@@ -1,6 +1,6 @@
 {
  "name": "zigbee2mqtt",
-  "version": "1.22.0",
+  "version": "1.28.4",
  "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
  "main": "index.js",
  "repository": {
@@ -8,7 +8,7 @@
    "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
  },
  "engines": {
-    "node": "^10 || ^12 || ^14 || ^15 || ^16"
+    "node": "^14 || ^16 || ^18 || ^19"
  },
  "keywords": [
    "xiaomi",
@@ -35,55 +35,56 @@
  },
  "homepage": "https://koenkk.github.io/zigbee2mqtt",
  "dependencies": {
-    "ajv": "^8.6.3",
+    "ajv": "^8.11.2",
    "bind-decorator": "^1.0.11",
-    "core-js": "^3.18.2",
+    "connect-gzip-static": "2.1.1",
    "debounce": "^1.2.1",
-    "deep-object-diff": "^1.1.0",
+    "deep-object-diff": "^1.1.9",
    "fast-deep-equal": "^3.1.3",
-    "finalhandler": "^1.1.2",
+    "finalhandler": "^1.2.0",
    "git-last-commit": "^1.0.1",
-    "humanize-duration": "^3.27.0",
+    "humanize-duration": "^3.27.3",
    "js-yaml": "^4.1.0",
    "json-stable-stringify-without-jsonify": "^1.0.1",
+    "jszip": "^3.10.1",
    "mkdir-recursive": "^0.4.0",
-    "moment": "^2.29.1",
-    "mqtt": "4.2.8",
+    "moment": "^2.29.4",
+    "mqtt": "4.3.7",
    "object-assign-deep": "^0.4.0",
    "rimraf": "^3.0.2",
-    "semver": "^7.3.5",
-    "serve-static": "^1.14.1",
-    "source-map-support": "^0.5.20",
-    "winston": "^3.3.3",
-    "winston-syslog": "^2.4.4",
-    "ws": "^8.2.3",
-    "zigbee-herdsman": "0.13.164",
-    "zigbee-herdsman-converters": "14.0.303",
-    "zigbee2mqtt-frontend": "0.6.30"
+    "semver": "^7.3.8",
+    "source-map-support": "^0.5.21",
+    "uri-js": "^4.4.1",
+    "winston": "^3.8.2",
+    "winston-syslog": "^2.7.0",
+    "winston-transport": "^4.5.0",
+    "ws": "^8.11.0",
+    "zigbee-herdsman": "0.14.76",
+    "zigbee-herdsman-converters": "14.0.681-hotfix.0",
+    "zigbee2mqtt-frontend": "0.6.119"
  },
  "devDependencies": {
-    "@babel/core": "^7.15.5",
-    "@babel/plugin-proposal-decorators": "^7.15.4",
-    "@babel/preset-env": "^7.15.6",
-    "@babel/preset-typescript": "^7.15.0",
+    "@babel/core": "^7.20.2",
+    "@babel/plugin-proposal-decorators": "^7.20.2",
+    "@babel/preset-env": "^7.20.2",
+    "@babel/preset-typescript": "^7.18.6",
    "@types/debounce": "^1.2.1",
-    "@types/finalhandler": "^1.1.1",
-    "@types/humanize-duration": "^3.25.1",
-    "@types/jest": "^27.0.2",
-    "@types/js-yaml": "^4.0.3",
+    "@types/finalhandler": "^1.2.0",
+    "@types/humanize-duration": "^3.27.1",
+    "@types/jest": "^29.2.3",
+    "@types/js-yaml": "^4.0.5",
    "@types/object-assign-deep": "^0.4.0",
    "@types/rimraf": "^3.0.2",
-    "@types/serve-static": "^1.13.10",
-    "@types/ws": "^8.2.0",
-    "@typescript-eslint/eslint-plugin": "^4.32.0",
-    "@typescript-eslint/parser": "^4.32.0",
-    "babel-jest": "^27.2.4",
-    "eslint": "^7.32.0",
+    "@types/ws": "^8.5.3",
+    "@typescript-eslint/eslint-plugin": "^5.44.0",
+    "@typescript-eslint/parser": "^5.44.0",
+    "babel-jest": "^29.3.1",
+    "eslint": "^8.28.0",
    "eslint-config-google": "^0.14.0",
-    "eslint-plugin-jest": "^24.5.0",
-    "jest": "^27.2.4",
+    "eslint-plugin-jest": "^27.1.6",
+    "jest": "^29.3.1",
    "tmp": "^0.2.1",
-    "typescript": "^4.4.3"
+    "typescript": "^4.9.3"
  },
  "jest": {
    "coverageThreshold": {
@@ -6,5 +6,6 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');

 for (const line of text.split('\n')) {
    const model = zigbeeHerdsmanConverters.devices.find((d) => d.model === line);
+    if (!model) throw new Error(`${line} does not exist`);
    console.log(`- \`${line}\` ${model.vendor} ${model.description}`);
 }
@@ -0,0 +1,22 @@
+const assert = require('assert');
+const vm = require('vm');
+const fs = require('fs');
+const path = require('path');
+const filename = process.argv[2];
+const moduleCode = fs.readFileSync(filename);
+const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+const sandbox = {
+    require: require,
+    module: {},
+    console,
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
+    setImmediate,
+    clearImmediate,
+};
+vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+const converter = sandbox.module.exports;
+assert(!converter.toZigbee.includes(undefined));
+assert(!converter.fromZigbee.includes(undefined));
@@ -205,6 +205,20 @@ describe('Availability', () => {
        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
    });

+    it('Should stop pinging device when it is removed', async () => {
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(9));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', stringify({id: "bulb_color"}));
+        await flushPromises();
+
+        await advancedTime(utils.minutes(3));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
    it('Should allow to be disabled', async () => {
        settings.set(['availability'], false);
        await resetExtension();
@@ -213,9 +227,23 @@ describe('Availability', () => {
        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
    });

+    it('Should to enable availabilty for just one device', async () => {
+        settings.set(['availability'], false);
+        settings.set(['devices', devices.bulb_color.ieeeAddr, 'availability'], true);
+
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(11));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
    it('Should retrieve device state when it reconnects', async () => {
        MQTT.publish.mockClear();

+        const device = controller.zigbee.resolveEntity(devices.bulb_color.ieeeAddr);
+        controller.state.set(device, {state: 'OFF'})
+
        const endpoint = devices.bulb_color.getEndpoint(1);
        endpoint.read.mockClear();

@@ -228,13 +256,9 @@ describe('Availability', () => {
        expect(endpoint.read).toHaveBeenCalledTimes(0);
        await advancedTime(utils.seconds(2));

-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl',
-            ['colorMode', 'currentX', 'currentY', 'enhancedCurrentHue', 'currentSaturation', 'colorTemperature']);

-        // Should stop when one request fails
        endpoint.read.mockClear();
        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
        await flushPromises();
@@ -249,6 +273,8 @@ describe('Availability', () => {
        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({from: 'bulb_color', to: 'bulb_new_name'}));
        await flushPromises();

+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            null, {retain: true, qos: 0}, expect.any(Function));
        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_new_name/availability',
            'online', {retain: true, qos: 0}, expect.any(Function));
        await advancedTime(utils.hours(12));
@@ -256,6 +282,16 @@ describe('Availability', () => {
            'offline', {retain: true, qos: 0}, expect.any(Function));
    });

+    it('Should publish availabiltiy payload in JSON format', async () => {
+        settings.set(['advanced', 'legacy_availability_payload'], false);
+        await resetExtension();
+        MQTT.publish.mockClear();
+        await advancedTime(utils.hours(26));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            stringify({state: 'offline'}), {retain: true, qos: 0}, expect.any(Function));
+    });
+
    it('Deprecated - should allow to block via advanced.availability_blocklist', async () => {
        settings.set(['advanced', 'availability_blocklist'], [devices.bulb_color.ieeeAddr]);
        await resetExtension();
@@ -282,4 +318,21 @@ describe('Availability', () => {
        await advancedTime(utils.minutes(12));
        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
    });
+
+    it('Should publish availabilty for groups', async () => {
+        settings.set(['devices', devices.bulb_color_2.ieeeAddr, 'availability'], true);
+        await resetExtension();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        MQTT.publish.mockClear();
+        await advancedTime(utils.minutes(12));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+        MQTT.publish.mockClear();
+        devices.bulb_color_2.lastSeen = Date.now();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color_2});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/group_tradfri_remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
 });
@@ -588,6 +588,7 @@ describe('Bind', () => {
    it('Should poll bounded Hue bulb when receiving message from scene controller', async () => {
        const remote = zigbeeHerdsman.devices.bj_scene_switch;
        const data = {"action": "recall_2_row_1"};
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockImplementationOnce(() => {throw new Error('failed')});
        const payload = {data, cluster: 'genScenes', device: remote, endpoint: remote.getEndpoint(10), type: 'commandRecall', linkquality: 10, groupID: 0};
        await zigbeeHerdsman.events.message(payload);
        await flushPromises();
@@ -595,7 +596,7 @@ describe('Bind', () => {
        expect(debounce).toHaveBeenCalledTimes(3);
        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
-	expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("lightingColorCtrl", ["currentX", "currentY", "colorTemperature"]);
+	    expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("lightingColorCtrl", ["currentX", "currentY", "colorTemperature"]);
    });

    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote', async () => {
@@ -100,12 +100,12 @@ describe('Configure', () => {
        expectBulbConfigured();
    });

-    it('Should reconfigure reporting on reportingDisabled event', async () => {
+    it('Should reconfigure reporting on reconfigure event', async () => {
        expectBulbConfigured();
        const device = controller.zigbee.resolveEntity(zigbeeHerdsman.devices.bulb);
        mockClear(device.zh);
        expectBulbNotConfigured();
-        controller.eventBus.emitReportingDisabled({device});
+        controller.eventBus.emitReconfigure({device});
        await flushPromises();
        expectBulbConfigured();
    });
@@ -192,19 +192,7 @@ describe('Configure', () => {
        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
    });

-    it('Should not configure when interviewing', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        delete device.meta.configured;
-        device.interviewing = true;
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        await zigbeeHerdsman.events.lastSeenChanged({device});
-        await flushPromises();
-        expectRemoteNotConfigured();
-        device.interviewing = false;
-    });
-
-    it('Should configure when not interviewCompleted', async () => {
+    it('Should not configure when interview not completed', async () => {
        const device = zigbeeHerdsman.devices.remote;
        delete device.meta.configured;
        device.interviewCompleted = false;
@@ -212,7 +200,7 @@ describe('Configure', () => {
        mockClear(device);
        await zigbeeHerdsman.events.lastSeenChanged({device});
        await flushPromises();
-        expectRemoteConfigured();
+        expectRemoteNotConfigured();
        device.interviewCompleted = true;
    });

@@ -28,9 +28,10 @@ describe('Controller', () => {
        MQTT.restoreOnMock();
        zigbeeHerdsman.returnDevices.splice(0);
        mockExit = jest.fn();
+        data.writeDefaultConfiguration();
+        settings.reRead();
        controller = new Controller(jest.fn(), mockExit);
        mocksClear.forEach((m) => m.mockClear());
-        data.writeDefaultConfiguration();
        settings.reRead();
        data.writeDefaultState();
    });
@@ -43,7 +44,6 @@ describe('Controller', () => {
        await controller.start();
        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null, disableLED: false}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
        expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
@@ -120,6 +120,7 @@ describe('Controller', () => {
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", stringify({"brightness":255}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":'ON'}), {"qos": 0, "retain":false}, expect.any(Function));
    });

    it('Start controller should not publish cached states when disabled', async () => {
@@ -181,14 +182,6 @@ describe('Controller', () => {
        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
    });

-    it('Should remove non whitelisted devices on startup', async () => {
-        settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
-        await controller.start();
-        await flushPromises();
-        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
-        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
-    });
-
    it('Should remove device on blocklist on startup', async () => {
        settings.set(['blocklist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
        await controller.start();
@@ -197,14 +190,6 @@ describe('Controller', () => {
        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
    });

-    it('Should remove banned devices on startup', async () => {
-        settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
-        await controller.start();
-        await flushPromises();
-        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
-    });
-
    it('Start controller fails', async () => {
        zigbeeHerdsman.start.mockImplementationOnce(() => {throw new Error('failed')});
        await controller.start();
@@ -217,9 +202,10 @@ describe('Controller', () => {
        });
        await controller.start();
        await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith('MQTT failed to connect: addr not found');
+        expect(logger.error).toHaveBeenCalledWith('MQTT error: addr not found');
+        expect(logger.error).toHaveBeenCalledWith('MQTT failed to connect, exiting...');
        expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, false);
    });

    it('Start controller with permit join true', async () => {
@@ -229,13 +215,6 @@ describe('Controller', () => {
        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
    });

-    it('Start controller with disable_led', async () => {
-        settings.set(['serial', 'disable_led'], true);
-        await controller.start();
-        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.setLED).toHaveBeenCalledWith(false);
-    });
-
    it('Start controller with transmit power', async () => {
        settings.set(['experimental', 'transmit_power'], 14);
        await controller.start();
@@ -243,13 +222,13 @@ describe('Controller', () => {
        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledWith(14);
    });

-    it('Start controller and stop', async () => {
+    it('Start controller and stop with restart', async () => {
        await controller.start();
-        await controller.stop();
+        await controller.stop(true);
        expect(MQTT.end).toHaveBeenCalledTimes(1);
        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
        expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(0);
+        expect(mockExit).toHaveBeenCalledWith(0, true);
    });

    it('Start controller and stop', async () => {
@@ -259,7 +238,7 @@ describe('Controller', () => {
        expect(MQTT.end).toHaveBeenCalledTimes(1);
        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
        expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, false);
    });

    it('Start controller adapter disconnects', async () => {
@@ -270,7 +249,7 @@ describe('Controller', () => {
        expect(MQTT.end).toHaveBeenCalledTimes(1);
        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
        expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, false);
    });

    it('Handle mqtt message', async () => {
@@ -512,6 +491,46 @@ describe('Controller', () => {
        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
    });

+    it('Publish entity state attribute_json output filtered cache', async () => {
+        await controller.start();
+        settings.set(['advanced', 'output'], 'attribute_and_json');
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_cache'], ['linkquality']);
+        MQTT.publish.mockClear();
+
+        const device = controller.zigbee.resolveEntity('bulb');
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
+
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
+        await flushPromises();
+
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output filtered cache (device_options)', async () => {
+        await controller.start();
+        settings.set(['advanced', 'output'], 'attribute_and_json');
+        settings.set(['device_options', 'filtered_cache'], ['linkquality']);
+        MQTT.publish.mockClear();
+
+        const device = controller.zigbee.resolveEntity('bulb');
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
+
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
+        await flushPromises();
+
+        expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
    it('Publish entity state with device information', async () => {
        await controller.start();
        settings.set(['mqtt', 'include_device_information'], true);
@@ -592,7 +611,7 @@ describe('Controller', () => {
        logger.error.mockClear();
        controller.state.file = "/";
        await controller.state.save();
-        expect(logger.error).toHaveBeenCalledWith(`Failed to write state to '/' (EISDIR: illegal operation on a directory, open '/')`);
+        expect(logger.error).toHaveBeenCalledWith(expect.stringMatching(/Failed to write state to \'\/\'/));
    });

    it('Publish should not cache when set', async () => {
@@ -627,6 +646,27 @@ describe('Controller', () => {
        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
    });

+    it('Should republish retained messages on MQTT reconnect', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        MQTT.events['connect']();
+        await flushPromises();
+        jest.runOnlyPendingTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(13);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Should not republish retained messages on MQTT reconnect when retained message are sent', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        MQTT.events['connect']();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/state', 'online');
+        jest.runOnlyPendingTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/state', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
        settings.set(['mqtt', 'force_disable_retain'], true);
        await controller.mqtt.connect()
@@ -638,11 +678,13 @@ describe('Controller', () => {
    });

    it('Should disable legacy options on new network start', async () => {
-        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        settings.set(['homeassistant'], true);
+        settings.reRead();
+        expect(settings.get().homeassistant.legacy_entity_attributes).toBeTruthy();
        expect(settings.get().advanced.legacy_api).toBeTruthy();
        zigbeeHerdsman.start.mockReturnValueOnce('reset');
        await controller.start();
-        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().homeassistant.legacy_entity_attributes).toBeFalsy();
        expect(settings.get().advanced.legacy_api).toBeFalsy();
    });

@@ -652,9 +694,29 @@ describe('Controller', () => {
        await flushPromises();
        MQTT.publish.mockClear();
        const device = zigbeeHerdsman.devices.remote;
-        await zigbeeHerdsman.events.lastSeenChanged({device});
+        await zigbeeHerdsman.events.lastSeenChanged({device, reason: 'deviceAnnounce'});
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/remote', stringify({"brightness":255,"last_seen":1000}), { qos: 0, retain: true }, expect.any(Function));
    });
+
+    it('Should not publish last seen changes when reason is messageEmitted', async () => {
+        settings.set(['advanced', 'last_seen'], 'epoch');
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        await zigbeeHerdsman.events.lastSeenChanged({device, reason: 'messageEmitted'});
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Ignore messages from coordinator', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/9218
+        await controller.start();
+        const device = zigbeeHerdsman.devices.coordinator;
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'Coordinator', type 'attributeReport', cluster 'genBasic', data '{"modelId":null}' from endpoint 1, ignoring since it is from coordinator`);
+    });
 });
@@ -15,12 +15,13 @@ describe('Data', () => {
        it('Should return correct path when ZIGBEE2MQTT_DATA set', () => {
            const expected = tmp.dirSync().name;
            process.env.ZIGBEE2MQTT_DATA = expected;
-            data.__testingOnly_reload();
+            data.testingOnlyReload();
            const actual = data.getPath();
            expect(actual).toBe(expected);
            expect(data.joinPath('test')).toStrictEqual(path.join(expected, 'test'));
+            expect(data.joinPath('/test')).toStrictEqual(path.resolve(expected, '/test'));
            delete process.env.ZIGBEE2MQTT_DATA;
-            data.__testingOnly_reload();
+            data.testingOnlyReload();
        });
    });
 });
@@ -84,7 +84,12 @@ describe('User extensions', () => {
        MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
        await flushPromises();

-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/response/extension/save', stringify({"data":{},"error":"Unexpected identifier","status":"error"}), { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/response/extension/save', expect.any(String), { retain: false, qos: 0 }, expect.any(Function));
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload).toEqual(
+            expect.objectContaining({"data":{},"status":"error"})
+        );
+        expect(payload.error).toMatch("Unexpected identifier");
    });

    it('Removes user extension', async () => {
@@ -50,7 +50,7 @@ jest.mock('http', () => ({
    }),
 }));

-jest.mock("serve-static", () =>
+jest.mock("connect-gzip-static", () =>
    jest.fn().mockImplementation((path) => {
        mockNodeStatic.variables.path = path
        return mockNodeStatic.implementation
@@ -140,7 +140,7 @@ describe('Frontend', () => {
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb_color',
-            stringify({state: 'ON', linkquality: null, update_available: null}),
+            stringify({state: 'ON', power_on_behavior:null, linkquality: null, update_available: null, update: {state: null, installed_version: "unknown"}}),
            { retain: false, qos: 0 },
            expect.any(Function)
        );
@@ -151,7 +151,7 @@ describe('Frontend', () => {

        // Received message on socket
        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(1);
-        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', linkquality: null, update_available: null}}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', power_on_behavior:null, linkquality: null, update_available: null, update: {state: null, installed_version: "unknown"}}}));

        // Shouldnt set when not ready
        mockWSClient.implementation.send.mockClear();
@@ -469,6 +469,30 @@ describe('Groups', () => {
        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
    });

+    it('Should publish state change off if any lights within are still on when changed via device when off_state: last_member_state is used', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false, off_state: 'last_member_state'}
+        });
+        await resetExtension();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1, "zigbee2mqtt/group_1", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2, "zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
    it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
        const device_1 = zigbeeHerdsman.devices.bulb_color;
        const device_2 = zigbeeHerdsman.devices.bulb;
@@ -24,6 +24,7 @@ describe('HomeAssistant extension', () => {
    beforeEach(async () => {
        data.writeDefaultConfiguration();
        settings.reRead();
+        settings.set(['homeassistant'], true);
        data.writeEmptyState();
        controller.state.load();
        await resetExtension();
@@ -49,14 +50,22 @@ describe('HomeAssistant extension', () => {
    it('Should not have duplicate type/object_ids in a mapping', () => {
        const duplicated = [];
        require('zigbee-herdsman-converters').devices.forEach((d) => {
-            const mapping = extension._getMapping()[d.model];
+            const exposes = typeof d.exposes == 'function' ? d.exposes() : d.exposes;
+            const device = {definition: d, isDevice: () => true, options: {}, exposes: () => exposes};
+            const configs = extension.getConfigs(device);
            const cfg_type_object_ids = [];

-            mapping.forEach((c) => {
-                if (cfg_type_object_ids.includes(c['type'] + '/' + c['object_id'])) {
-                    duplicated.push(d.model);
+            configs.forEach((c) => {
+                const id = c['type'] + '/' + c['object_id'];
+                if (cfg_type_object_ids.includes(id)) {
+                    if (typeof d.exposes == 'function') {
+                        // A dynamic function must exposes all possible attributes for the docs
+                        console.warn(`${d.model} dynamic exposes contains duplicated ${id}`)
+                    } else {
+                        duplicated.push(d.model);
+                    }
                } else {
-                    cfg_type_object_ids.push(c['type'] + '/' + c['object_id']);
+                    cfg_type_object_ids.push(id);
                }
            });
        });
@@ -77,7 +86,7 @@ describe('HomeAssistant extension', () => {
            "device":{
               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
               "name":"ha_discovery_group",
-               "sw_version":version,
+               "sw_version": version,
            },
            "max_mireds": 454,
            "min_mireds": 250,
@@ -89,6 +98,15 @@ describe('HomeAssistant extension', () => {
               "xy",
               "color_temp"
            ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
            "unique_id":"9_light_zigbee2mqtt"
        };

@@ -105,7 +123,7 @@ describe('HomeAssistant extension', () => {
            "device":{
               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
               "name":"ha_discovery_group",
-               "sw_version":version,
+               "sw_version": version,
            },
            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
            "name":"ha_discovery_group",
@@ -135,7 +153,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -163,7 +181,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -190,7 +208,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -214,10 +232,11 @@ describe('HomeAssistant extension', () => {
            'name': 'weather_sensor_battery',
            'unique_id': '0x0017880104e45522_battery_zigbee2mqtt',
            'enabled_by_default': true,
+            'entity_category': 'diagnostic',
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -234,6 +253,7 @@ describe('HomeAssistant extension', () => {
        payload = {
            'icon': 'mdi:signal',
            'enabled_by_default': false,
+            'entity_category': 'diagnostic',
            'unit_of_measurement': 'lqi',
            'state_class': 'measurement',
            'value_template': '{{ value_json.linkquality }}',
@@ -244,7 +264,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -268,7 +288,7 @@ describe('HomeAssistant extension', () => {
                "manufacturer":"Xiaomi",
                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
                "name":"wall_switch_double",
-                "sw_version":version
+                "sw_version": null
            },
            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
            "name":"wall_switch_double_left",
@@ -296,7 +316,7 @@ describe('HomeAssistant extension', () => {
                "manufacturer":"Xiaomi",
                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
                "name":"wall_switch_double",
-                "sw_version":version
+                "sw_version": null
            },
            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
            "name":"wall_switch_double_right",
@@ -330,7 +350,7 @@ describe('HomeAssistant extension', () => {
                "manufacturer":"IKEA",
                "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                "name":"bulb",
-                "sw_version":version,
+                "sw_version": null,
            },
            "effect":true,
            "effect_list":[
@@ -383,7 +403,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -410,7 +430,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -437,7 +457,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -470,7 +490,7 @@ describe('HomeAssistant extension', () => {
                device: {
                    manufacturer: 'Not from Xiaomi',
                    model: 'custom model',
-                }
+                },
            },
            friendly_name: 'weather_sensor',
            retain: false,
@@ -522,7 +542,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'custom model',
                'manufacturer': 'Not from Xiaomi',
            },
@@ -539,6 +559,75 @@ describe('HomeAssistant extension', () => {
        );
    });

+    it('Should discover devices with overriden name', async () => {
+        settings.set(['devices', '0x0017880104e45522'], {
+            homeassistant: {
+                name: "Weather Sensor",
+            },
+            friendly_name: 'weather_sensor',
+            retain: false,
+        })
+
+        await resetExtension();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'state_class': 'measurement',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor',
+            'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
+            'name': 'Weather Sensor temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'Weather Sensor',
+                'sw_version': null,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            'enabled_by_default': true,
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            'unit_of_measurement': '%',
+            'device_class': 'humidity',
+            'state_class': 'measurement',
+            'value_template': '{{ value_json.humidity }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor',
+            'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
+            'name': 'Weather Sensor humidity',
+            'unique_id': '0x0017880104e45522_humidity_zigbee2mqtt',
+            'enabled_by_default': true,
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'Weather Sensor',
+                'sw_version': null,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/humidity/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
    it('Should discover devices with overriden user configuration affecting type and object_id', async () => {
        settings.set(['devices', '0x0017880104e45541'], {
            friendly_name: 'my_switch',
@@ -550,8 +639,7 @@ describe('HomeAssistant extension', () => {
                light: {
                    type: 'this should be ignored',
                    name: 'my_light_name_override'
-                }
-
+                },
            },
        })

@@ -570,7 +658,7 @@ describe('HomeAssistant extension', () => {
              "manufacturer": "Xiaomi",
              "model": "Aqara single key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG04LM)",
              "name": "my_switch",
-              "sw_version": version
+              "sw_version": null
            },
            "json_attributes_topic": "zigbee2mqtt/my_switch",
            "name": "my_light_name_override",
@@ -647,7 +735,7 @@ describe('HomeAssistant extension', () => {
                  "zigbee2mqtt_0x0017880104e45548"
               ],
               "name":"fan",
-               "sw_version":version,
+               "sw_version": null,
               "model":"Universal wink enabled white ceiling fan premier remote control (99432)",
               "manufacturer":"Hampton Bay"
            },
@@ -666,32 +754,36 @@ describe('HomeAssistant extension', () => {
        let payload;

        payload = {
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
-            "away_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/away_mode",
-            "away_mode_state_template":"{{ value_json.away_mode }}",
-            "away_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "action_template":"{% set values = {None:None,'idle':'off','heat':'heating','cool':'cooling','fan_only':'fan'} %}{{ values[value_json.running_state] }}",
+            "action_topic":"zigbee2mqtt/TS0601_thermostat",
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
            "current_temperature_template":"{{ value_json.local_temperature }}",
            "current_temperature_topic":"zigbee2mqtt/TS0601_thermostat",
            "device":{
-               "identifiers":[
-                  "zigbee2mqtt_0x0017882104a44559"
-               ],
-               "manufacturer":"TuYa",
-               "model":"Radiator valve with thermostat (TS0601_thermostat)",
-               "name":"TS0601_thermostat",
-               "sw_version":  version,
+                "identifiers":[
+                    "zigbee2mqtt_0x0017882104a44559"
+                ],
+                "manufacturer":"TuYa",
+                "model":"Radiator valve with thermostat (TS0601_thermostat)",
+                "name":"TS0601_thermostat",
+                "sw_version":null
            },
-            "hold_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
-            "hold_modes":[
-               "schedule",
-               "manual",
-               "boost",
-               "complex",
-               "comfort",
-               "eco"
+            "preset_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
+            "preset_modes":[
+                "schedule",
+                "manual",
+                "boost",
+                "complex",
+                "comfort",
+                "eco",
+                "away"
            ],
-            "hold_state_template":"{{ value_json.preset }}",
-            "hold_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "preset_mode_value_template":"{{ value_json.preset }}",
+            "preset_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
            "json_attributes_topic":"zigbee2mqtt/TS0601_thermostat",
            "max_temp":"35",
            "min_temp":"5",
@@ -699,7 +791,9 @@ describe('HomeAssistant extension', () => {
            "mode_state_template":"{{ value_json.system_mode }}",
            "mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
            "modes":[
-               "heat", "auto", "off"
+                "heat",
+                "auto",
+                "off"
            ],
            "name":"TS0601_thermostat",
            "temp_step":0.5,
@@ -727,6 +821,10 @@ describe('HomeAssistant extension', () => {
            set_position_topic: 'zigbee2mqtt/smart vent/set',
            set_position_template: '{ "position": {{ position }} }',
            position_template: '{{ value_json.position }}',
+            state_topic: 'zigbee2mqtt/smart vent',
+            value_template: `{{ value_json.state }}`,
+            state_open: 'OPEN',
+            state_closed: 'CLOSE',
            json_attributes_topic: 'zigbee2mqtt/smart vent',
            name: 'smart vent',
            unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
@@ -734,7 +832,7 @@ describe('HomeAssistant extension', () => {
            {
                identifiers: [ 'zigbee2mqtt_0x0017880104e45551' ],
                name: 'smart vent',
-                sw_version: version,
+                sw_version: null,
                model: 'Smart vent (SV01)',
                manufacturer: 'Keen Home'
            },
@@ -747,6 +845,39 @@ describe('HomeAssistant extension', () => {
            { retain: true, qos: 0 },
            expect.any(Function),
        );
+
+        payload = {
+            "availability": [{ "topic": "zigbee2mqtt/bridge/state" }],
+            "command_topic": "zigbee2mqtt/zigfred_plus/l6/set",
+            "device": {
+                "identifiers": ["zigbee2mqtt_0xf4ce368a38be56a1"],
+                "manufacturer": "Siglis",
+                "model": "zigfred plus smart in-wall switch (ZFP-1A-CH)",
+                "name": "zigfred_plus",
+                "sw_version": null
+            },
+            "json_attributes_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "name": "zigfred_plus_l6",
+            "position_template": "{{ value_json.position }}",
+            "position_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "set_position_template": "{ \"position_l6\": {{ position }} }",
+            "set_position_topic": "zigbee2mqtt/zigfred_plus/l6/set",
+            "state_closed": "CLOSE",
+            "state_open": "OPEN",
+            "state_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "tilt_command_topic": "zigbee2mqtt/zigfred_plus/l6/set/tilt",
+            "tilt_status_template": "{{ value_json.tilt }}",
+            "tilt_status_topic": "zigbee2mqtt/zigfred_plus/l6",
+            "unique_id": "0xf4ce368a38be56a1_cover_l6_zigbee2mqtt",
+            "value_template": "{{ value_json.state }}"
+        }
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/cover/0xf4ce368a38be56a1/cover_l6/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
    });

    it('Should discover devices with custom homeassistant_discovery_topic', async () => {
@@ -768,7 +899,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -825,7 +956,7 @@ describe('HomeAssistant extension', () => {
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb_color',
-            stringify({"color":{"hue": 0, "saturation": 100, "h": 0, "s": 100}, "color_mode": "hs", "linkquality": null, "state": null, "update_available": null}),
+            stringify({"color":{"hue": 0, "saturation": 100, "h": 0, "s": 100}, "color_mode": "hs", "linkquality": null, "state": null, "update_available": null, "power_on_behavior":null, "update": {"state": null, "installed_version": "unknown"}}),
            { retain: false, qos: 0 },
            expect.any(Function),
        );
@@ -841,7 +972,7 @@ describe('HomeAssistant extension', () => {
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb_color',
-            stringify({"color": {"x": 0.4576,"y": 0.41}, "color_mode": "xy", "linkquality": null,"state": null, "update_available": null}),
+            stringify({"color": {"x": 0.4576,"y": 0.41}, "color_mode": "xy", "linkquality": null,"state": null, "update_available": null, "power_on_behavior":null, "update": {"state": null, "installed_version": "unknown"}}),
            { retain: false, qos: 0 },
            expect.any(Function),
        );
@@ -857,7 +988,7 @@ describe('HomeAssistant extension', () => {
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb_color',
-            stringify({"linkquality": null,"state": "ON", "update_available": null}),
+            stringify({"linkquality": null,"state": "ON", "update_available": null, "power_on_behavior": null, "update": {"state": null, "installed_version": "unknown"}}),
            { retain: false, qos: 0 },
            expect.any(Function),
        );
@@ -895,7 +1026,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -919,6 +1050,19 @@ describe('HomeAssistant extension', () => {
        await flushPromises();
    });

+    it('Should discover when options change', async () => {
+        const device = controller.zigbee.resolveEntity(zigbeeHerdsman.devices.bulb);
+        MQTT.publish.mockClear();
+        controller.eventBus.emitEntityOptionsChanged({entity: device, from: {}, to: {'test': 123}});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            `homeassistant/light/${device.ID}/light/config`,
+            expect.any(String),
+            expect.any(Object),
+            expect.any(Function),
+        );
+    });
+
    it('Should send all status when home assistant comes online (default topic)', async () => {
        data.writeDefaultState();
        extension.state.load();
@@ -932,16 +1076,22 @@ describe('HomeAssistant extension', () => {
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null, "update": {"state": null, "installed_version": "unknown"}}),
            { retain: true, qos: 0 },
            expect.any(Function)
        );
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/remote',
-            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null}),
+            stringify({"action":null,"action_duration":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null, "update": {"state": null, "installed_version": "unknown"}}),
            { retain: true, qos: 0 },
            expect.any(Function)
        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/group_1',
+            stringify({"state":'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
    });

    it('Should send all status when home assistant comes online', async () => {
@@ -956,13 +1106,13 @@ describe('HomeAssistant extension', () => {
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null, "update_available": null, "update": {"state": null, "installed_version": "unknown"}}),
            { retain: true, qos: 0 },
            expect.any(Function)
        );
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/remote',
-            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null}),
+            stringify({"action":null,"action_duration":null,"battery":null,"brightness":255,"linkquality":null, "update_available": null, "update": {"state": null, "installed_version": "unknown"}}),
            { retain: true, qos: 0 },
            expect.any(Function)
        );
@@ -1012,7 +1162,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -1078,6 +1228,8 @@ describe('HomeAssistant extension', () => {
        MQTT.publish.mockClear();
        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":true}));
        await flushPromises();
+        jest.runOnlyPendingTimers();
+        await flushPromises();

        const payload = {
            'unit_of_measurement': '°C',
@@ -1092,7 +1244,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor_renamed',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -1126,7 +1278,7 @@ describe('HomeAssistant extension', () => {
                        "zigbee2mqtt_0x0017880104e45522"
                    ],
                    "name":"weather_sensor_renamed",
-                    "sw_version": version,
+                    "sw_version": null,
                    "model":"Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)",
                    "manufacturer":"Xiaomi"
                }
@@ -1136,6 +1288,61 @@ describe('HomeAssistant extension', () => {
        );
    });

+    it('Should refresh discovery when group is renamed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/rename', stringify({"from": "ha_discovery_group", "to": "ha_discovery_group_new","homeassistant_rename":true}));
+        await flushPromises();
+        jest.runOnlyPendingTimers();
+        await flushPromises();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group_new/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group_new",
+               "sw_version": version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group_new",
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "name":"ha_discovery_group_new",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group_new",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
    it('Shouldnt refresh discovery when device is renamed and homeassistant_rename is false', async () => {
        MQTT.publish.mockClear();
        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":false}));
@@ -1161,7 +1368,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor_renamed',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -1180,7 +1387,7 @@ describe('HomeAssistant extension', () => {
        const payload = {
            "payload_on":true,
            "payload_off":false,
-            "value_template":"{{ value_json.update_available}}",
+            "value_template":`{{ value_json['update']['state'] == "available" }}`,
            "enabled_by_default": false,
            "state_topic":"zigbee2mqtt/bulb",
            "json_attributes_topic":"zigbee2mqtt/bulb",
@@ -1191,11 +1398,13 @@ describe('HomeAssistant extension', () => {
                    "zigbee2mqtt_0x000b57fffec6a5b2"
                ],
                "name":"bulb",
-                'sw_version': version,
+                'sw_version': null,
                "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                "manufacturer":"IKEA"
            },
            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            'device_class': 'update',
+            'entity_category': 'diagnostic'
        };

        expect(MQTT.publish).toHaveBeenCalledWith(
@@ -1208,7 +1417,7 @@ describe('HomeAssistant extension', () => {

    it('Should discover trigger when click is published', async () => {
        const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(5);
+        expect(discovered.length).toBe(7);
        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/click/config');
        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');

@@ -1230,7 +1439,7 @@ describe('HomeAssistant extension', () => {
                    "zigbee2mqtt_0x0017880104e45520"
                ],
                "name":"button",
-                "sw_version": version,
+                "sw_version": null,
                "model":"Aqara wireless switch (WXKG11LM)",
                "manufacturer":"Xiaomi"
            }
@@ -1254,7 +1463,7 @@ describe('HomeAssistant extension', () => {
                    "zigbee2mqtt_0x0017880104e45520"
                ],
                "name":"button",
-                "sw_version": version,
+                "sw_version": null,
                "model":"Aqara wireless switch (WXKG11LM)",
                "manufacturer":"Xiaomi"
            }
@@ -1283,21 +1492,21 @@ describe('HomeAssistant extension', () => {

        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/button',
-            stringify({action: "single", click: "single", battery: null, linkquality: null, voltage: null}),
+            stringify({action: "single", click: "single", battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null}),
            { retain: false, qos: 0 },
            expect.any(Function),
        );

        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/button',
-            stringify({action: "", battery: null, linkquality: null, voltage: null, click: null}),
+            stringify({action: "", battery: null, linkquality: null, voltage: null, click: null, power_outage_count: null, device_temperature: null}),
            { retain: false, qos: 0 },
            expect.any(Function),
        );

        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/button',
-            stringify({click: "", action: null, battery: null, linkquality: null, voltage: null}),
+            stringify({click: "", action: null, battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null}),
            { retain: false, qos: 0 },
            expect.any(Function),
        );
@@ -1384,7 +1593,7 @@ describe('HomeAssistant extension', () => {
        await resetExtension();

        const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(4);
+        expect(discovered.length).toBe(6);
        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');
        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/battery/config');
        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/linkquality/config');
@@ -1395,7 +1604,7 @@ describe('HomeAssistant extension', () => {
        await resetExtension();

        const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(3);
+        expect(discovered.length).toBe(5);
        expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/click/config');
        expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/action/config');

@@ -1418,7 +1627,7 @@ describe('HomeAssistant extension', () => {
                    "zigbee2mqtt_0x0017880104e45520"
                ],
                "name":"button",
-                "sw_version": version,
+                "sw_version": null,
                "model":"Aqara wireless switch (WXKG11LM)",
                "manufacturer":"Xiaomi"
            }
@@ -1440,7 +1649,7 @@ describe('HomeAssistant extension', () => {

        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/button',
-            stringify({action: "single", "battery":null,"linkquality":null,"voltage":null}),
+            stringify({action: "single", "battery":null,"linkquality":null,"voltage":null, "power_outage_count": null, "device_temperature": null}),
            { retain: false, qos: 0 },
            expect.any(Function),
        );
@@ -1477,33 +1686,15 @@ describe('HomeAssistant extension', () => {
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(4);
        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', click: null, battery: null, linkquality: null, voltage: null});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', click: null, battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null});
        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', click: null, battery: null, linkquality: null, voltage: null});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', click: null, battery: null, linkquality: null, voltage: null, power_outage_count: null, device_temperature: null});
        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/action_single/config');
        expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/action');
    });

-    it('Load Home Assistant mapping from external converters', async () => {
-        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
-        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        await resetExtension();
-
-        const homeassistantSwitch = {
-            type: 'switch',
-            object_id: 'switch',
-            discovery_payload: {
-                payload_off: 'OFF',
-                payload_on: 'ON',
-                value_template: '{{ value_json.state }}',
-                command_topic: true,
-            },
-        };
-        expect(extension._getMapping()['external_converters_device_1']).toEqual([homeassistantSwitch]);
-    });
-
    it('Should clear outdated configs', async () => {
        // Non-existing group -> clear
        MQTT.publish.mockClear();
@@ -1609,7 +1800,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
            },
@@ -1638,7 +1829,7 @@ describe('HomeAssistant extension', () => {
            "device":{
               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
               "name":"ha_discovery_group",
-               "sw_version":version,
+               "sw_version": version,
            },
            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
            "max_mireds": 454,
@@ -1650,6 +1841,15 @@ describe('HomeAssistant extension', () => {
               "xy",
               "color_temp"
            ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
            "unique_id":"9_light_zigbee2mqtt"
         };

@@ -1661,8 +1861,53 @@ describe('HomeAssistant extension', () => {
        );
    });

+    it('Should discover with json availability payload value_template', async () => {
+        settings.set(['advanced', 'legacy_availability_payload'], false);
+        await resetExtension();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state","value_template":'{{ value_json.state }}'}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version": version,
+            },
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
    it('Should discover last_seen when enabled', async () => {
-        settings.set(['advanced', 'last_seen'], ' epoch');
+        settings.set(['advanced', 'last_seen'], 'ISO_8601');
        await resetExtension();

        const payload = {
@@ -1678,7 +1923,7 @@ describe('HomeAssistant extension', () => {
               "manufacturer":"IKEA",
               "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
               "name":"bulb",
-               "sw_version":version
+               "sw_version": null
            },
            "enabled_by_default":false,
            "icon":"mdi:clock",
@@ -1686,7 +1931,9 @@ describe('HomeAssistant extension', () => {
            "name":"bulb last seen",
            "state_topic":"zigbee2mqtt/bulb",
            "unique_id":"0x000b57fffec6a5b2_last_seen_zigbee2mqtt",
-            "value_template":"{{ value_json.last_seen }}"
+            "value_template":"{{ value_json.last_seen }}",
+            "device_class": "timestamp",
+            "entity_category": "diagnostic"
         };

        expect(MQTT.publish).toHaveBeenCalledWith(
@@ -1718,7 +1965,7 @@ describe('HomeAssistant extension', () => {
            'device': {
                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                'name': 'weather_sensor',
-                'sw_version': version,
+                'sw_version': null,
                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                'manufacturer': 'Xiaomi',
                'configuration_url': 'http://zigbee.mqtt/#/device/0x0017880104e45522/info'
@@ -56,7 +56,7 @@ describe('Bridge legacy', () => {
    it('Should allow whitelist', async () => {
        const bulb_color = zigbeeHerdsman.devices.bulb_color;
        const bulb = zigbeeHerdsman.devices.bulb;
-        expect(settings.get().whitelist).toStrictEqual([]);
+        expect(settings.get().passlist).toStrictEqual([]);
        MQTT.publish.mockClear();
        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb_color');
        await flushPromises();
@@ -68,7 +68,7 @@ describe('Bridge legacy', () => {
        );

        MQTT.publish.mockClear()
-        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr]);
+        expect(settings.get().passlist).toStrictEqual([bulb_color.ieeeAddr]);
        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledWith(
@@ -79,10 +79,10 @@ describe('Bridge legacy', () => {
        );

        MQTT.publish.mockClear()
-        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        expect(settings.get().passlist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
        await flushPromises();
-        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        expect(settings.get().passlist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
        expect(MQTT.publish).toHaveBeenCalledTimes(0);
    });

@@ -366,7 +366,7 @@ describe('Bridge legacy', () => {
        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
        const device = zigbeeHerdsman.devices.bulb_color;
        device.removeFromNetwork.mockClear();
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
        await flushPromises();
        MQTT.publish.mockClear();
        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
@@ -381,14 +381,14 @@ describe('Bridge legacy', () => {
            expect.any(Function)
        );
        expect(controller.state.state).toStrictEqual({});
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
    });

    it('Should allow to force remove device', async () => {
        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
        const device = zigbeeHerdsman.devices.bulb_color;
        device.removeFromDatabase.mockClear();
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
        await flushPromises();
        MQTT.publish.mockClear();
        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove', 'bulb_color');
@@ -403,13 +403,13 @@ describe('Bridge legacy', () => {
            expect.any(Function)
        );
        expect(controller.state.state).toStrictEqual({});
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
    });

-    it('Should allow to ban device', async () => {
+    it('Should allow to block device', async () => {
        const device = zigbeeHerdsman.devices.bulb_color;
        device.removeFromNetwork.mockClear();
-        expect(settings.get().ban.length).toBe(0);
+        expect(settings.get().blocklist.length).toBe(0);
        await flushPromises();
        MQTT.publish.mockClear();
        MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
@@ -423,7 +423,7 @@ describe('Bridge legacy', () => {
            {qos: 0, retain: false},
            expect.any(Function)
        );
-        expect(settings.get().ban).toStrictEqual(['0x000b57fffec6a5b3']);
+        expect(settings.get().blocklist).toStrictEqual(['0x000b57fffec6a5b3']);
    });

    it('Shouldnt crash when removing non-existing device', async () => {
@@ -25,6 +25,7 @@ describe('Logger', () => {

    it('Create log directory', () => {
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
        const dirs = fs.readdirSync(dir.name);
        expect(dirs.length).toBe(1);
@@ -32,6 +33,7 @@ describe('Logger', () => {

    it('Should cleanup', () => {
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();

        for (const d of fs.readdirSync(dir.name)) {
@@ -49,6 +51,7 @@ describe('Logger', () => {

    it('Should not cleanup when there is no timestamp set', () => {
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
        for (let i = 30; i < 40; i++) {
            fs.mkdirSync(path.join(dir.name, `log_${i}`));
@@ -62,6 +65,7 @@ describe('Logger', () => {

    it('Set and get log level', () => {
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
        logger.setLevel('debug');
        expect(logger.getLevel()).toBe('debug');
@@ -74,23 +78,26 @@ describe('Logger', () => {
        }

        const logger = require('../lib/util/logger').default;
-        expect(logger.winston.transports.length).toBe(2);
+        logger.init();
+        expect(logger.winston().transports.length).toBe(2);
        logger.addTransport(new DummyTransport());
-        expect(logger.winston.transports.length).toBe(3);
+        expect(logger.winston().transports.length).toBe(3);
    });

    it('Set and get log level warn <-> warning', () => {
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
        logger.setLevel('warn');
-        expect(logger.winston.transports[0].level).toBe('warning');
+        expect(logger.winston().transports[0].level).toBe('warning');
        expect(logger.getLevel()).toBe('warn');
    });

    it('Logger should be console and file by default', () => {
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
        expect(pipes.length).toBe(2);
        expect(pipes[0].constructor.name).toBe('Console');
        expect(pipes[0].silent).toBe(false);
@@ -101,8 +108,9 @@ describe('Logger', () => {
    it('Logger can be file only', () => {
        settings.set(['advanced', 'log_output'], ['file']);
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
        expect(pipes.length).toBe(2);
        expect(pipes[0].constructor.name).toBe('Console');
        expect(pipes[0].silent).toBe(true);
@@ -113,8 +121,9 @@ describe('Logger', () => {
    it('Logger can be console only', () => {
        settings.set(['advanced', 'log_output'], ['console']);
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
        expect(pipes.constructor.name).toBe('Console');
        expect(pipes.silent).toBe(false);
    });
@@ -122,8 +131,9 @@ describe('Logger', () => {
    it('Logger can be nothing', () => {
        settings.set(['advanced', 'log_output'], []);
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
        expect(pipes.constructor.name).toBe('Console');
        expect(pipes.silent).toBe(true);
    });
@@ -131,8 +141,9 @@ describe('Logger', () => {
    it('Should allow to disable log rotation', () => {
        settings.set(['advanced', 'log_rotation'], false);
        const logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
-        const pipes = logger.winston._readableState.pipes;
+        const pipes = logger.winston()._readableState.pipes;
        expect(pipes[1].constructor.name).toBe('File');
        expect(pipes[1].maxFiles).toBeNull();
        expect(pipes[1].tailable).toBeFalsy();
@@ -142,6 +153,7 @@ describe('Logger', () => {
    it('Should allow to symlink logs to current directory', () => {
        settings.set(['advanced', 'log_symlink_current'], true);
        let logger = require('../lib/util/logger').default;
+        logger.init();
        logger.logOutput();
        expect(fs.readdirSync(dir.name).includes('current')).toBeTruthy()

@@ -151,23 +163,24 @@ describe('Logger', () => {

    it('Log', () => {
        const logger = require('../lib/util/logger').default;
-        const warn = jest.spyOn(logger.winston, 'warning');
+        logger.init();
+        const warn = jest.spyOn(logger.winston(), 'warning');
        logger.warn('warn');
        expect(warn).toHaveBeenCalledWith('warn');

-        const debug = jest.spyOn(logger.winston, 'debug');
+        const debug = jest.spyOn(logger.winston(), 'debug');
        logger.debug('debug');
        expect(debug).toHaveBeenCalledWith('debug');

-        const warning = jest.spyOn(logger.winston, 'warning');
+        const warning = jest.spyOn(logger.winston(), 'warning');
        logger.warning('warning');
        expect(warning).toHaveBeenCalledWith('warning');

-        const info = jest.spyOn(logger.winston, 'info');
+        const info = jest.spyOn(logger.winston(), 'info');
        logger.info('info');
        expect(info).toHaveBeenCalledWith('info');

-        const error = jest.spyOn(logger.winston, 'error');
+        const error = jest.spyOn(logger.winston(), 'error');
        logger.error('error');
        expect(error).toHaveBeenCalledWith('error');
    });
@@ -102,7 +102,7 @@ describe('Networkmap', () => {
        let call = MQTT.publish.mock.calls[0];
        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');

-        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
+        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, power_on_behavior, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, power_outage_count, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
        expect(JSON.parse(call[1])).toStrictEqual(expected);

        /**
@@ -276,7 +276,7 @@ describe('Networkmap', () => {
        let call = MQTT.publish.mock.calls[0];
        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');

-        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, power_on_behavior, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, power_outage_count, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
        const actual = JSON.parse(call[1]);
        expect(actual).toStrictEqual(expected);
    });
@@ -48,6 +48,8 @@ describe('On event', () => {
        expect(call[0]).toBe('start')
        expect(call[1]).toStrictEqual({})
        expect(call[2]).toBe(device);
+        expect(call[3]).toStrictEqual(settings.getDevice(device.ieeeAddr));
+        expect(call[4]).toStrictEqual({});
    });

    it('Should call with stop event', async () => {
@@ -66,7 +68,7 @@ describe('On event', () => {
        await zigbeeHerdsman.events.deviceAnnounce({device});
        await flushPromises();
        expect(mockOnEvent).toHaveBeenCalledTimes(1);
-        expect(mockOnEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr));
+        expect(mockOnEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr), {});
    });

    it('Should call index onEvent with zigbee event', async () => {
@@ -74,6 +76,6 @@ describe('On event', () => {
        await zigbeeHerdsman.events.deviceAnnounce({device});
        await flushPromises();
        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device);
+        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr), {});
    });
 });
@@ -1,3 +1,5 @@
+const path = require('path');
+
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
@@ -7,10 +9,19 @@ const Controller = require('../lib/controller');
 const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const stringify = require('json-stable-stringify-without-jsonify');
+const zigbeeOTA = require('zigbee-herdsman-converters/lib/ota/zigbeeOTA');
+
+const spyUseIndexOverride = jest.spyOn(zigbeeOTA, 'useIndexOverride');
+

 describe('OTA update', () => {
    let controller;

+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'OTAUpdate');
+        await controller.enableDisableExtension(true, 'OTAUpdate');
+    }
+
    const mockClear = (mapped) => {
        mapped.ota.updateToLatest = jest.fn();
        mapped.ota.isUpdateAvailable = jest.fn();
@@ -20,7 +31,7 @@ describe('OTA update', () => {
        data.writeDefaultConfiguration();
        settings.reRead();
        data.writeDefaultConfiguration();
-        settings.set(['advanced', 'ikea_ota_use_test_url'], true);
+        settings.set(['ota', 'ikea_ota_use_test_url'], true);
        settings.reRead();
        jest.useFakeTimers();
        controller = new Controller(jest.fn(), jest.fn());
@@ -70,10 +81,11 @@ describe('OTA update', () => {
        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
-        expect(device.save).toHaveBeenCalledTimes(1);
-        expect(device.dateCode).toBe('20190102');
-        expect(device.softwareBuildID).toBe(2);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+        expect(logger.info).toHaveBeenCalledWith(`Device 'bulb' was updated from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190103","softwareBuildID":3}'`);
+        expect(device.save).toHaveBeenCalledTimes(2);
+        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhen': 'immediate'});
+        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhen': 'active'});
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb',
            stringify({"update_available":false,"update":{"state":"updating","progress":0}}),
@@ -91,7 +103,7 @@ describe('OTA update', () => {
        );
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bridge/response/device/ota_update/update',
-            stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
+            stringify({"data":{"from":{"date_code":"20190101","software_build_id":1},"id":"bulb","to":{"date_code":"20190103","software_build_id":3}},"status":"ok"}),
            {retain: false, qos: 0}, expect.any(Function)
        );
        expect(MQTT.publish).toHaveBeenCalledWith(
@@ -246,7 +258,7 @@ describe('OTA update', () => {
        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x98});

        // Should not request again when device asks again after a short time
        await zigbeeHerdsman.events.message(payload);
@@ -265,6 +277,28 @@ describe('OTA update', () => {
        );
    });

+    it('Should respond with NO_IMAGE_AVAILABLE when update available request fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('Nothing to find here')})
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x98});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"idle"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
    it('Should check for update when device requests it and it is not available', async () => {
        const device = zigbeeHerdsman.devices.bulb;
        device.endpoints[0].commandResponse.mockClear();
@@ -279,7 +313,7 @@ describe('OTA update', () => {
        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x98});
        expect(MQTT.publish).toHaveBeenCalledWith(
            'zigbee2mqtt/bulb',
            stringify({"update_available":false,"update":{"state":"idle"}}),
@@ -348,11 +382,11 @@ describe('OTA update', () => {
        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+        expect(logger.info).toHaveBeenCalledWith(`Device 'bulb' was updated from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190103","softwareBuildID":3}'`);        
        expect(logger.error).toHaveBeenCalledTimes(0);
-        expect(device.save).toHaveBeenCalledTimes(1);
-        expect(device.dateCode).toBe('20190102');
-        expect(device.softwareBuildID).toBe(2);
+        expect(device.save).toHaveBeenCalledTimes(2);
+        expect(endpoint.read).toHaveBeenCalledWith('genBasic', ['dateCode', 'swBuildId'], {'sendWhen': 'immediate'});
    });

    it('Legacy api: Should handle when OTA update fails', async () => {
@@ -433,4 +467,16 @@ describe('OTA update', () => {
        await flushPromises();
        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
    });
+
+    it('Set zigbee_ota_override_index_location', async () => {
+        settings.set(['ota', 'zigbee_ota_override_index_location'], 'local.index.json');
+        await resetExtension();
+        expect(spyUseIndexOverride).toHaveBeenCalledWith(path.join(data.mockDir, 'local.index.json'));
+        spyUseIndexOverride.mockClear();
+        
+        settings.set(['ota', 'zigbee_ota_override_index_location'], 'http://my.site/index.json');
+        await resetExtension();
+        expect(spyUseIndexOverride).toHaveBeenCalledWith('http://my.site/index.json');
+        spyUseIndexOverride.mockClear();
+    });
 });
@@ -367,7 +367,7 @@ describe('Publish', () => {
        await MQTT.events.message('zigbee2mqtt/thermostat_group/set', stringify({child_lock: 'LOCK'}));
        await flushPromises();
        expect(group.command).toHaveBeenCalledTimes(1);
-        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 7, length_hi: 0, length_lo: 1, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "dataRequest", {dpValues: [{data: [1], datatype: 1, dp: 7}], seq: expect.any(Number)}, {disableDefaultResponse: true});
    });

    it('Should publish messages to groups with on and brightness', async () => {
@@ -507,6 +507,18 @@ describe('Publish', () => {
        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
    });

+    it('Should handle get with multiple cover endpoints', async () => {
+        const device = zigbeeHerdsman.devices.zigfred_plus;
+        const endpoint11 = device.getEndpoint(11);
+        const endpoint12 = device.getEndpoint(12);
+        await MQTT.events.message('zigbee2mqtt/zigfred_plus/get', stringify({state_l6: '', state_l7: ''}));
+        await flushPromises();
+        expect(endpoint11.read).toHaveBeenCalledTimes(1);
+        expect(endpoint11.read).toHaveBeenCalledWith('closuresWindowCovering', ['currentPositionLiftPercentage']);
+        expect(endpoint12.read).toHaveBeenCalledTimes(1);
+        expect(endpoint12.read).toHaveBeenCalledWith('closuresWindowCovering', ['currentPositionLiftPercentage']);
+    });
+
    it('Should log error when device has no such endpoint (via topic)', async () => {
        const device = zigbeeHerdsman.devices.QBKG03LM;
        const endpoint2 = device.getEndpoint(2);
@@ -919,11 +931,11 @@ describe('Publish', () => {
        expect(endpoint.command).toHaveBeenCalledTimes(3);
        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 0}, {}]);
-        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
        expect(MQTT.publish).toHaveBeenCalledTimes(3);
        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 1}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
    });

    it('Should turn device off when brightness 0 is send with transition', async () => {
@@ -938,11 +950,11 @@ describe('Publish', () => {
        expect(endpoint.command).toHaveBeenCalledTimes(3);
        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
-        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
        expect(MQTT.publish).toHaveBeenCalledTimes(3);
        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 1}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
    });

    it('Should allow to set color via hue and saturation', async () => {
@@ -1294,7 +1306,7 @@ describe('Publish', () => {
        await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', stringify({state: 'ON'}));
        await flushPromises();
        expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 3, length_hi: 0, length_lo: 1, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "dataRequest", {dpValues: [{data: [1], datatype: 1, dp: 3}], seq: expect.any(Number)}, {disableDefaultResponse: true});
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});
@@ -84,7 +84,7 @@ describe('Receive', () => {
        await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20, state: 'OPEN'}), {retain: false, qos: 0}, expect.any(Function));

        // Inverted
        MQTT.publish.mockClear();
@@ -92,7 +92,7 @@ describe('Receive', () => {
        await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80, state: 'OPEN'}), {retain: false, qos: 0}, expect.any(Function));
    });

    it('Should allow to disable the legacy integration', async () => {
@@ -242,70 +242,15 @@ describe('Receive', () => {
        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
    });

-    it('WSDCGQ11LM pressure precision from non ZCL properties', async () => {
-        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
-        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
-
-        MQTT.publish.mockClear();
-        let payload = {data: {"65281":{"1":2985,"4":5032,"5":9,"6":[0,1],"10":0,"100":2345,"101":4608,"102":91552}}, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5});
-
-        MQTT.publish.mockClear();
-        payload = {data: {"16":9354,"20":-1,"measuredValue":915}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4});
-    });
-
-    it('Should handle a zigbee message with voltage 3010', async () => {
+    it('Should handle a zigbee message with voltage 2990', async () => {
        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 3010}}
+        const data = {'65281': {'1': 2990}}
        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
        await zigbeeHerdsman.events.message(payload);
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2850', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2850}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2650', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2650}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
-    it('Should handle a zigbee message with voltage 2000', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
-        const data = {'65281': {'1': 2000}}
-        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 93, voltage: 2990});
        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
    });

@@ -317,7 +262,7 @@ describe('Receive', () => {
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'temperature': 19});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'device_temperature': 19, 'power_outage_count': 34});
        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
    });

@@ -369,20 +314,6 @@ describe('Receive', () => {
        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
    });

-    it('Should handle forwarded Xiaomi messages', async () => {
-        const device = zigbeeHerdsman.devices.ZNCZ02LM;
-        const payload = {data: {measuredValue: -85}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 6539};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/weather_sensor',
-            stringify({temperature: -0.85}),
-            {"qos": 1, "retain": false},
-            expect.any(Function),
-        )
-    });
-
    it('Should handle messages from Xiaomi router devices', async () => {
        const device = zigbeeHerdsman.devices.ZNCZ02LM;
        const data = {onOff: 1};
@@ -422,7 +353,7 @@ describe('Receive', () => {
        await zigbeeHerdsman.events.message(payload);
        await flushPromises();
        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID' and manufacturer name 'notSupportedMfg'`);
-        expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+        expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/advanced/support-new-devices/01_support_new_devices.html`);
        expect(MQTT.publish).toHaveBeenCalledTimes(0);
    });

@@ -440,7 +371,7 @@ describe('Receive', () => {
    it('Should handle a command', async () => {
        const device = zigbeeHerdsman.devices.E1743;
        const data = {};
-        const payload = {data, cluster: 'genLevelCtrl', device, endpoint: device.getEndpoint(1), type: 'commandStopWithOnOff', linkquality: 10};
+        const payload = {data, cluster: 'genLevelCtrl', device, endpoint: device.getEndpoint(1), type: 'commandStopWithOnOff', linkquality: 10, meta: {zclTransactionSequenceNumber: 1}};
        await zigbeeHerdsman.events.message(payload);
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
@@ -456,10 +387,10 @@ describe('Receive', () => {
        const oldNow = Date.now;
        Date.now = jest.fn()
        Date.now.mockReturnValue(new Date(150));
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.message({...payload, meta: {zclTransactionSequenceNumber: 2}});
        await flushPromises();
        Date.now.mockReturnValue(new Date(200));
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.message({...payload, meta: {zclTransactionSequenceNumber: 3}});
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(2);
        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
@@ -488,7 +419,7 @@ describe('Receive', () => {
        const data = {instantaneousDemand:496,currentSummDelivered:[0,6648]}

        const SP600_NEW = zigbeeHerdsman.devices.SP600_NEW;
-        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_NEW, endpoint: SP600_NEW.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_NEW, endpoint: SP600_NEW.getEndpoint(1), type: 'attributeReport', linkquality: 10, meta: {zclTransactionSequenceNumber: 1}});
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
@@ -496,7 +427,7 @@ describe('Receive', () => {

        MQTT.publish.mockClear();
        const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
-        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_OLD, endpoint: SP600_OLD.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_OLD, endpoint: SP600_OLD.getEndpoint(1), type: 'attributeReport', linkquality: 10, meta: {zclTransactionSequenceNumber: 2}});
        await flushPromises();
        expect(MQTT.publish).toHaveBeenCalledTimes(1);
        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');
@@ -39,6 +39,7 @@ describe('Settings', () => {

    beforeEach(() => {
        remove(configurationFile);
+        remove(secretFile);
        remove(devicesFile);
        remove(groupsFile);
        clearEnvironmentVariables();
@@ -66,20 +67,42 @@ describe('Settings', () => {
    it('Should apply environment variables', () => {
        process.env['ZIGBEE2MQTT_CONFIG_SERIAL_DISABLE_LED'] = 'true';
        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_SOFT_RESET_TIMEOUT'] = 1;
-        process.env['ZIGBEE2MQTT_CONFIG_EXPERIMENTAL_OUTPUT'] = 'csvtest';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_OUTPUT'] = 'attribute_and_json';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_LOG_OUTPUT'] = '["console"]';
        process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
        process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
+        process.env['ZIGBEE2MQTT_CONFIG_MQTT_SERVER'] = 'testserver';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_NETWORK_KEY'] = 'GENERATE';
+        process.env['ZIGBEE2MQTT_CONFIG_DEVICES'] = 'devices.yaml';
+
+        const contentDevices = {
+            '0x00158d00018255df': {
+                friendly_name: '0x00158d00018255df',
+                retain: false,
+            },
+        };

        write(configurationFile, {});
+        write(devicesFile, contentDevices);
+        expect(settings.validate()).toStrictEqual([]);
+
        const s = settings.get();
        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
-        expected.devices = {};
+        expected.devices = {
+            '0x00158d00018255df': {
+                friendly_name: '0x00158d00018255df',
+                retain: false,
+            },
+        };
        expected.groups = {};
        expected.serial.disable_led = true;
        expected.advanced.soft_reset_timeout = 1;
-        expected.experimental.output = 'csvtest';
+        expected.advanced.log_output = ["console"];
+        expected.advanced.output = 'attribute_and_json';
        expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
        expected.mqtt.base_topic = 'testtopic';
+        expected.mqtt.server = 'testserver';
+        expected.advanced.network_key = 'GENERATE';

        expect(s).toStrictEqual(expected);
    });
@@ -150,6 +173,7 @@ describe('Settings', () => {
        write(configurationFile, contentConfiguration);

        const expected = {
+            base_topic: 'zigbee2mqtt',
            include_device_information: false,
            force_disable_retain: false,
            password: "mysecretpassword",
@@ -170,6 +194,49 @@ describe('Settings', () => {
        expect(read(secretFile)).toStrictEqual({...contentSecret, username: 'test123', network_key: [1,2,3,4]});
    });

+    it('Should read ALL secrets form a separate file', () => {
+        const contentConfiguration = {
+            mqtt: {
+                server: '!secret server',
+                user: '!secret username',
+                password: '!secret.yaml password',
+            },
+            advanced: {
+                network_key: '!secret network_key',
+            }
+        };
+
+        const contentSecret = {
+            server: 'my.mqtt.server',
+            username: 'mysecretusername',
+            password: 'mysecretpassword',
+            network_key: [1,2,3],
+        };
+
+        write(secretFile, contentSecret, false);
+        write(configurationFile, contentConfiguration);
+
+        const expected = {
+            base_topic: 'zigbee2mqtt',
+            include_device_information: false,
+            force_disable_retain: false,
+            password: "mysecretpassword",
+            server: "my.mqtt.server",
+            user: "mysecretusername",
+        };
+
+        expect(settings.get().mqtt).toStrictEqual(expected);
+        expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
+
+        settings.testing.write();
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual(contentSecret);
+
+        settings.set(['mqtt', 'server'], 'not.secret.server');
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual({...contentSecret, server: 'not.secret.server'});
+    });
+
    it('Should read devices form a separate file', () => {
        const contentConfiguration = {
            devices: 'devices.yaml',
@@ -672,14 +739,6 @@ describe('Settings', () => {
        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
    });

-    it('Should ban devices', () => {
-        write(configurationFile, {});
-        settings.banDevice('0x123');
-        expect(settings.get().ban).toStrictEqual(['0x123']);
-        settings.banDevice('0x1234');
-        expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
-    });
-
    it('Should add devices to blocklist', () => {
        write(configurationFile, {});
        settings.blockDevice('0x123');
@@ -705,6 +764,18 @@ describe('Settings', () => {
        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
    });

+    it('Configuration shouldnt be valid when invalid QOS value is used', async () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            devices: {'0x0017880104e45519': {friendly_name: 'myname', retain: false, qos: 3}},
+        });
+
+        settings.reRead();
+
+        const error = `QOS for 'myname' not valid, should be 0, 1 or 2 got 3`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
    it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
        write(configurationFile, {
            ...minimalConfig,
@@ -742,15 +813,15 @@ describe('Settings', () => {
        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
    });

-    it('Configuration friendly name cannot contain null char', async () => {
+    it('Configuration friendly name cannot contain control char', async () => {
        write(configurationFile, {
            ...minimalConfig,
-            devices: {'0x0017880104e45519': {friendly_name: 'blaa/blaa' + String.fromCharCode(0), retain: false}},
+            devices: {'0x0017880104e45519': {friendly_name: 'blaa/blaa\u009f', retain: false}},
        });

        settings.reRead();

-        const error = `friendly_name is not allowed to contain null char`;
+        const error = `friendly_name is not allowed to contain control char`;
        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
    });

@@ -830,4 +901,78 @@ describe('Settings', () => {
        const after = fs.statSync(configurationFile).mtimeMs;
        expect(before).toBe(after);
    });
+
+    it('Frontend config', () => {
+        write(configurationFile, {...minimalConfig,
+            frontend: true, 
+        });
+
+        settings.reRead();
+        expect(settings.get().frontend).toStrictEqual({port: 8080, auth_token: false, host: '0.0.0.0'})
+    });
+
+    it('Baudrate config', () => {
+        write(configurationFile, {...minimalConfig,
+            advanced: {baudrate: 20}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().serial.baudrate).toStrictEqual(20)
+    });
+
+    it('ikea_ota_use_test_url config', () => {
+        write(configurationFile, {...minimalConfig,
+            advanced: {ikea_ota_use_test_url: true}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().ota.ikea_ota_use_test_url).toStrictEqual(true)
+    });
+
+    it('transmit_power config', () => {
+        write(configurationFile, {...minimalConfig,
+            experimental: {transmit_power: 1337}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().advanced.transmit_power).toStrictEqual(1337)
+    });
+
+    it('output config', () => {
+        write(configurationFile, {...minimalConfig,
+            experimental: {output: 'json'}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().advanced.output).toStrictEqual('json')
+    });
+
+    it('Baudrartsctste config', () => {
+        write(configurationFile, {...minimalConfig,
+            advanced: {rtscts: true}, 
+        });
+
+        settings.reRead();
+        expect(settings.get().serial.rtscts).toStrictEqual(true)
+    });
+
+    it('Deprecated: Home Assistant config', () => {
+        write(configurationFile, {...minimalConfig,
+            homeassistant: {discovery_topic: 'new'}, 
+            advanced: {homeassistant_discovery_topic: 'old', homeassistant_status_topic: 'olds'},
+        });
+
+        settings.reRead();
+        expect(settings.get().homeassistant).toStrictEqual({discovery_topic: 'new', legacy_entity_attributes: true, legacy_triggers: true, status_topic: 'olds'})
+    });
+
+    it('Deprecated: ban/whitelist config', () => {
+        write(configurationFile, {...minimalConfig,
+            ban: ['ban'], whitelist: ['whitelist'], passlist: ['passlist'], blocklist: ['blocklist']
+        });
+
+        settings.reRead();
+        expect(settings.get().blocklist).toStrictEqual(['blocklist', 'ban'])
+        expect(settings.get().passlist).toStrictEqual(['passlist', 'whitelist'])
+    });
 });
@@ -21,7 +21,8 @@ function writeDefaultConfiguration() {
        devices: {
            "0x000b57fffec6a5b2": {
                retain: true,
-                friendly_name: "bulb"
+                friendly_name: "bulb",
+                description: "this is my bulb",
            },
            "0x0017880104e45517": {
                retain: true,
@@ -152,6 +153,23 @@ function writeDefaultConfiguration() {
            '0x0017880104e43559': {
                friendly_name: 'U202DST600ZB'
            },
+            '0xf4ce368a38be56a1': {
+                retain: false,
+                friendly_name: 'zigfred_plus',
+                front_surface_enabled: 'true',
+                dimmer_1_enabled: 'true',
+                dimmer_1_dimming_enabled: 'true',
+                dimmer_2_enabled: 'true',
+                dimmer_2_dimming_enabled: 'true',
+                dimmer_3_enabled: 'true',
+                dimmer_3_dimming_enabled: 'true',
+                dimmer_4_enabled: 'true',
+                dimmer_4_dimming_enabled: 'true',
+                cover_1_enabled: 'true',
+                cover_1_tilt_enabled: 'true',
+                cover_2_enabled: 'true',
+                cover_2_tilt_enabled: 'true',
+            },
            '0x0017880104e44559': {
                friendly_name: '3157100_thermostat',
            },
@@ -242,6 +260,9 @@ const defaultState = {
    "0x0017880104e45517": {
        "brightness": 255
    },
+    "1": {
+        'state': 'ON',
+    }
 }

 function getDefaultState() {
@@ -12,6 +12,7 @@ const callTransports = (level, message) => {
 }

 const mock = {
+    init: jest.fn(),
    info: jest.fn().mockImplementation((msg) => callTransports('info', msg)),
    warn: jest.fn().mockImplementation((msg) => callTransports('warn', msg)),
    error: jest.fn().mockImplementation((msg) => callTransports('error', msg)),
@@ -23,6 +24,7 @@ const mock = {
    setLevel: (newLevel) => {level = newLevel},
    getLevel: () => level,
    setTransportsEnabled: (value) => {transportsEnabled = value},
+    end: jest.fn(),
 };

 jest.mock('../../lib/util/logger', () => (mock));
@@ -6,6 +6,7 @@ const mock = {
    subscribe: jest.fn(),
    reconnecting: false,
    on: jest.fn(),
+    stream: {setMaxListeners: jest.fn()}
 };

 const mockConnect = jest.fn().mockReturnValue(mock);
@@ -27,7 +27,11 @@ const clusters = {
    'lightingColorCtrl': 768,
    'closuresWindowCovering': 258,
    'hvacThermostat': 513,
+    'msIlluminanceMeasurement': 1024,
    'msTemperatureMeasurement': 1026,
+    'msRelativeHumidity': 1029,
+    'msSoilMoisture': 1032,
+    'msCO2': 1037
 }

 class Endpoint {
@@ -133,6 +137,7 @@ const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,41
 const ZNCZ02LM = new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug");
 const GLEDOPTO_2ID = new Device('Router', '0x0017880104e45724', 6540,4151, [new Endpoint(11, [0,3,4,5,6,8,768], [], '0x0017880104e45724', [], {}, [], 49246, 528), new Endpoint(12, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 260, 258), new Endpoint(13, [4096], [4096], '0x0017880104e45724', [], {}, [], 49246, 57694), new Endpoint(15, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 49246, 256)], true, "Mains (single phase)", 'GL-C-007', false, 'GLEDOPTO');
 const QBKG03LM = new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2');
+const zigfred_plus = new Device('Router', '0xf4ce368a38be56a1', 6589, 0x129C, [new Endpoint(5, [0, 3, 4, 5, 6, 8, 0x0300, 0xFC42], [0xFC42], '0xf4ce368a38be56a1'), new Endpoint(7, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(8, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(9, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(10, [0, 3, 4, 5, 6, 8], [], '0xf4ce368a38be56a1'), new Endpoint(11, [0, 3, 4, 5, 0x0102], [], '0xf4ce368a38be56a1'), new Endpoint(12, [0, 3, 4, 5, 0x0102], [], '0xf4ce368a38be56a1')], true, "Mains (single phase)", 'zigfred plus', false, 'Siglis');

 const groups = {
    'group_1': new Group(1, []),
@@ -188,7 +193,8 @@ const devices = {
    'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
    'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
    'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
-    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
+    'zigfred_plus': zigfred_plus,
+    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100', false, 'Centralite'),
    'J1': new Device('Router', '0x0017880104a44559', 6543,4151, [new Endpoint(1, [], [], '0x0017880104a44559')], true, "Mains (single phase)", 'J1 (5502)'),
    'TS0601_thermostat': TS0601_thermostat,
    'external_converter_device': new Device('EndDevice', '0x0017880104e45511', 1114, 'external', [new Endpoint(1, [], [], '0x0017880104e45511')], false, null, 'external_converter_device' ),
@@ -197,7 +203,7 @@ const devices = {
    'temperature_sensor': new Device('EndDevice', '0x0017880104e45561', 6544,4151, [new Endpoint(1, [0,3,4,1026], [])], true, "Battery", "temperature.sensor"),
    'heating_actuator': new Device('Router', '0x0017880104e45562', 6545,4151, [new Endpoint(1, [0,3,4,513], [1026])], true, "Mains (single phase)", "heating.actuator"),
    'bj_scene_switch': new Device('EndDevice', '0xd85def11a1002caa', 50117, 4398, [new Endpoint(10, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa', [{target: bulb_color_2.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 6, name: 'genOnOff'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 768, name: 'lightingColorCtrl'}},]), new Endpoint(11, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa')], true, 'Battery', 'RB01', false, 'Busch-Jaeger', '20161222', '1.2.0'),
-
+    'GW003-AS-IN-TE-FC': new Device('Router', '0x0017548104a44669', 6545,4699, [new Endpoint(1, [3], [0,3,513,514], '0x0017548104a44669')], true, "Mains (single phase)", 'Adapter Zigbee FUJITSU'),
 }

 const mock = {
@@ -207,15 +213,16 @@ const mock = {
    touchlinkScan: jest.fn(),
    touchlinkIdentify: jest.fn(),
    start: jest.fn(),
+    backup: jest.fn(),
    isStopping: jest.fn(),
    permitJoin: jest.fn(),
+    addInstallCode: jest.fn(),
    getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
    getNetworkParameters: jest.fn().mockReturnValue({panID: 0x162a, extendedPanID: [0, 11, 22], channel: 15}),
    on: (type, handler) => {
        events[type] = handler;
    },
    stop: jest.fn(),
-    setLED: jest.fn(),
    getDevices: jest.fn().mockImplementation(() => {
        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr));
    }),
@@ -18,7 +18,6 @@ echo "Creating backup of configuration..."
 cp -R data data-backup

 echo "Updating..."
-git checkout HEAD -- npm-shrinkwrap.json
 git pull

 echo "Installing dependencies..."