1.21.2

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.dockerignore

@@ -1,7 +1,17 @@
-node_modules
-.travis*
+.eslintignore
+.eslintrc.js
 .git
+.github
 .gitignore
-*.png
-firmware
-docker/Dockerfile*
+.idea
+.npmrc
+.travis*
+.travis.yml
+README.md
+docker/Dockerfile*
+images
+node_modules
+scripts
+test
+coverage
+update.sh

.eslintignore

@@ -1 +1,2 @@
-node_modules/*
+node_modules/*
+test/

.eslintrc.js

@@ -0,0 +1,45 @@
+
+module.exports = {
+    'env': {
+        'jest/globals': true,
+        'es6': true,
+        'node': true,
+    },
+    'extends': ['eslint:recommended', 'google', 'plugin:jest/recommended', 'plugin:jest/style'],
+    'parserOptions': {
+        'ecmaVersion': 2018,
+        'sourceType': 'module',
+    },
+    'rules': {
+        'require-jsdoc': 'off',
+        'indent': ['error', 4],
+        'max-len': ['error', {'code': 120}],
+        'no-prototype-builtins': 'off',
+    },
+    'plugins': [
+        'jest',
+    ],
+    'overrides': [{
+        files: ['*.ts'],
+        parser: '@typescript-eslint/parser',
+        plugins: ['@typescript-eslint'],
+        extends: ['plugin:@typescript-eslint/recommended'],
+        parserOptions: {
+            project: './tsconfig.json',
+        },
+        rules: {
+            '@typescript-eslint/await-thenable': 'error',
+            '@typescript-eslint/ban-ts-comment': 'off',
+            '@typescript-eslint/explicit-function-return-type': 'error',
+            '@typescript-eslint/no-empty-function': 'off',
+            '@typescript-eslint/no-explicit-any': 'error',
+            '@typescript-eslint/no-unused-vars': 'error',
+            '@typescript-eslint/semi': ['error'],
+            'array-bracket-spacing': ['error', 'never'],
+            'indent': ['error', 4],
+            'max-len': ['error', {'code': 120}],
+            'no-return-await': 'error',
+            'object-curly-spacing': ['error', 'never'],
+        },
+    }],
+};

.eslintrc.json

@@ -1,21 +0,0 @@
-{
-    "env": {
-        "jest/globals": true,
-        "es6": true,
-        "node": true
-    },
-    "extends": ["eslint:recommended", "google", "plugin:jest/recommended", "plugin:jest/style"],
-    "parserOptions": {
-        "ecmaVersion": 2018,
-        "sourceType": "module"
-    },
-    "rules": {
-        "require-jsdoc": "off",
-        "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }],
-        "no-prototype-builtins": "off"
-    },
-    "plugins": [
-        "jest"
-    ]
-}

.github/FUNDING.yml

@@ -1,2 +1,4 @@
 github: [koenkk]
-custom: https://www.paypal.me/koenkk
+custom:
+  - https://www.paypal.me/koenkk
+  - https://www.buymeacoffee.com/koenkk

.github/ISSUE_TEMPLATE.md

@@ -1,23 +1,38 @@
 <!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2MQTT version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
 Did you read the FAQ?
-https://www.zigbee2mqtt.io/information/FAQ.html
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2MQTT fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
 
 Unsupported device?
-https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
 
-A question?
-Read the documentation: https://www.zigbee2mqtt.io/
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
 
-Bug report:
-- First make sure you are on the latest version of zigbee2mqtt.
-- Make sure you are running the latest CC253X firmware from https://github.com/koenkk/z-stack-firmware.
-- Provide a clear description of the problem.
+Bug report?
 - If applicable, provide steps how to reproduce the problem.
-- Provide a log with `log_level` set to `debug`, see https://www.zigbee2mqtt.io/configuration/configuration.html
-- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
 -->
 
 <!--Start your bug report below this line-->
+
 # Bug Report
 ## What happened
 
@@ -26,5 +41,6 @@ Bug report:
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-zigbee2mqtt version:
-CC253X firmware version:
+Zigbee2MQTT version:
+Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
+Adapter firmware version:

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,17 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Questions/discussion
+    url: https://github.com/Koenkk/zigbee2mqtt/discussions/new
+    about: Ask questions, discuss about devices or show things you made
+  - name: Frontend issues
+    url: https://github.com/nurikk/z2m-frontend/issues
+    about: Issues related to the frontend.
+  - name: Home Assistant addon issues
+    url: https://github.com/zigbee2mqtt/hassio-zigbee2mqtt/issues
+    about: Issues related to the Home Assistant addon.
+  - name: FAQ
+    url: https://www.zigbee2mqtt.io/information/FAQ.html
+    about: Frequently asked questions.
+  - name: Support Chat
+    url: https://discord.gg/NyseBeK
+    about: Chat for feedback, questions and troubleshooting.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature request
+assignees: ''
+
+---
+
+## Is your feature request related to a problem? Please describe
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new_device_support.md

@@ -0,0 +1,21 @@
+---
+name: New device support request
+about: Request support for a new device
+title: ''
+labels: new device support
+assignees: ''
+
+---
+
+<!--
+VERY IMPORTANT: Before submitting a request, first (try to) follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Dumping just the info about this device without actually attemping adding support yourself will likely lead into this issue being ignored.
+Clearly indicate what you tried and where you got stuck.
+-->
+
+## Information about the device + link
+..
+
+## data/database.db entry of the device
+..

.github/ISSUE_TEMPLATE/problem_report.md

@@ -0,0 +1,54 @@
+---
+name: Problem report
+about: Create a report to help us improve
+title: ''
+labels: problem
+assignees: ''
+
+---
+
+<!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2MQTT version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
+Did you read the FAQ?
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2MQTT fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
+
+Unsupported device?
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
+
+Bug report?
+- If applicable, provide steps how to reproduce the problem.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
+-->
+
+<!--Start your bug report below this line-->
+
+## What happened
+
+## What did you expect to happen
+
+## How to reproduce it (minimal and precise)
+
+## Debug info
+Zigbee2MQTT version:
+Adapter hardware: CC2531, CC2530, CC26X2R1, CC1352P-2, Conbee II
+Adapter firmware version:

.github/branch-switcher.yml

@@ -0,0 +1 @@
+preferredBranch: dev

.github/stale.yml

@@ -1,13 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
-# Label to use when marking an issue as stale
-staleLabel: stale
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false

.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 14
+        registry-url: https://registry.npmjs.org/
+    - name: Get NodeJS Version
+      id: nodeversion
+      run: |
+        echo "::set-output name=version::$(node --version)"
+    - name: Restore node_modules cache
+      id: cache-node-modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+    - name: Install dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      run: npm ci
+    - name: Build
+      run: npm run build
+    - name: Test
+      run: npm run test-with-coverage
+    - name: Lint
+      run: npm run eslint
+    - name: Docker login
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
+    - name: Docker login ghcr.io
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u koenkk --password-stdin
+    - name: Docker setup - QEMU
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: all
+    - name: Docker setup - Buildx
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
+    - name: Docker build dev
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        docker buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest-dev -t ghcr.io/koenkk/zigbee2mqtt:latest-dev \
+          .
+    - name: Docker build release
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: |
+        TAG="$(git describe --tags)"
+        docker buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" -t ghcr.io/koenkk/zigbee2mqtt:latest -t "ghcr.io/koenkk/zigbee2mqtt:$TAG" \
+          .
+    - name: Publish to npm
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: npm publish
+      env:
+        NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN }}
+    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        curl -s \
+          --request POST \
+          --header "Content-Type: application/json" \
+          --data '{"definition": {"id": 9}}' \
+          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
+          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"

.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: "Close stale issues/pull requests"
+on:
+  schedule:
+  - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        stale-pr-message: 'This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        days-before-stale: 30
+        days-before-close: 7
+        operations-per-run: 500
+        exempt-issue-labels: 'breaking change'

.github/workflows/update_deps.yml

@@ -0,0 +1,25 @@
+name: Update dependencies
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "0 0 * * 0"
+
+jobs:
+  update_deps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 14
+    - run: rm -rf npm-shrinkwrap.json
+    - run: npx npm-check-updates -u
+    - run: npm install && npm shrinkwrap --dev
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update dependencies
+        branch: "update_deps"
+        title: Update dependencies

.github/workflows/update_frontend.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-frontend
+on:
+  repository_dispatch:
+    types: update_frontend
+
+jobs:
+  update_frontend:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 14
+    - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
+        branch: "update_frontend"
+        title: Update zigbee-frontend to ${{ github.event.client_payload.version }}

.github/workflows/update_zh.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman
+on:
+  repository_dispatch:
+    types: update_zh
+
+jobs:
+  update_zh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 14
+    - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
+        branch: "update_zh"
+        title: Update zigbee-herdsman to ${{ github.event.client_payload.version }}

.github/workflows/update_zhc.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman-converters
+on:
+  repository_dispatch:
+    types: update_zhc
+
+jobs:
+  update_zhc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 14
+    - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
+        branch: "update_zhc"
+        title: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}

.gitignore

@@ -11,6 +11,9 @@ pids
 *.seed
 *.pid.lock
 
+# Compiled source
+dist/*
+
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 
@@ -65,4 +68,15 @@ data/state.json
 data/log
 data-backup/
 data/coordinator_backup.json
-data/.storage
+data/.storage
+data/database.db.backup
+data/extension
+
+# MacOS indexing file
+.DS_Store
+
+# IDE specific folders
+.idea
+
+# Ignore config
+data/*.yaml

.npmignore

@@ -0,0 +1,27 @@
+#tests
+test
+coverage
+
+#build tools
+.travis.yml
+.jenkins.yml
+.codeclimate.yml
+.github
+babel.config.js
+
+#linters
+.jscsrc
+.jshintrc
+.eslintrc*
+.dockerignore
+.eslintignore
+.gitignore
+
+#editor settings
+.vscode
+
+#src
+lib
+docs
+docker
+images

.travis.yml

@@ -1,33 +0,0 @@
-language: node_js
-
-sudo: required
-
-services:
-  - docker
-
-node_js:
-  - "10"
-
-before_install:
-  - "echo '{\"experimental\": true}' | sudo tee /etc/docker/daemon.json"
-  - sudo apt update -y
-  - sudo apt install --only-upgrade docker-ce -y
-  - sudo service docker restart
-
-install:
-  - npm install
-
-before_script:
-  - npm test
-  - npm run eslint
-
-script:
-  - ".travis/docker.sh"
-  - ".travis/trigger-downstream.sh"
-
-env:
-  global:
-    - DOCKER_USERNAME=koenkk
-    - secure: q5W9+mHz13PslLkTxuYNXivOau+N5OyB53sqgb+krOCPlbN9+gvi2Av9SribVd5jp2yx2O9rV/mr8eS0M2ykvBJ3UsIsCXiFL1dEPmsz1wlFAnW3kbD2EGhGFEV0dWRflSlA+5iQtcMPvbYu2PAOChUUoY7NNlhcz2EC+XXUapvvvaDwA1oSdtX+m61qyREshZb8B57DyXhziy6i/xjhFa84C0mR/iGFRWpZgcxmrNFH+/XDhSIpdq2KshhDWhjyag3ukAsTxTpqzLh1rFRlczjJEn0CcHG4WHsWHUFn9KK2lvT/YCDP8w0URnR1bCr8nyxGZJ2DLW6AzxZtJfWjvbeZYjO2R3WeHpzp6Jw6Qz5pG3gmu2FgeFpZ3FTEGtJ/Ev0/pjkY2Msx9mItzUJ7D8hY2noQZMPASar6oXEDYlkomx7gHHla0MPv3PKSJg5LX6Ybl137GIa6MLYYyz6SEKfyRUQhy3Itqrb/ID/ZP0UuLgyyDfShb8A6XOnsVqwZAsOPtU2UPnD1vsVXhjkpflK9+7QjL5m9MSqfAhg6qtwNsswl1KSiyhmdldGhZW5OfUasDxZKHyx7bsS1BhNnLi/iy4ifDh6KSr64aUW4iM0X99gkU+662Hgf73QpIG+oP41r30iEN0unTcRrFQDIbTSSapbA95bwUCBF1CgQC20=
-    - secure: em3lBVXvZ3kOVI4ipVlh93NIU6guLvyzHMmsLDuxhc92VFURqlYu3iXTTNEnknwyE31mziAkVFnmj+3OENd/VfzP01d4KKaYAQrbn2AnOMvcIIExLfaFU5QqbW1RMwRBIifhvrmboKt07pFsUy3Rghs80SwNSFDfLFoMhFMz+NdIZfJGW8LJy+gUPmz10YGzqh/chreowqOemgBBoswCGwIh1xB1KNejew3bQllnwQYGXcGuTDwDT4Wz+ObxlFt7YR/4n4oJY8jQYRPolM9qHQAGyn/Cn0TL3c8JqiBpVc1hm2OhYSDqvK0xf5IKsD+kIfFJPGa7nF2cwmhKdUNYR47oue3KB5tdrtDhwqUooTglvBOSXUIbthUR9Xu40ziyd5tHy0ixaLnDRGuKIIvEnXlcE6CgApoe3y2kHYa5dY+eUKxWwvH7b5hG+u82dYvw3VYpCHAnDQVgUzz4F63qZ2WZhcH0W3PF3/ytVAHyMu2vAMSrSXGi5xgM6el7Gp7SFVTFXBurbOaoJ3ZwwSEDP5d8mDyk1d8thmkQFL9P6iWtNnXOzjv4GIliEDWQg76BX6MLD3n3fnP0aq+RVmv+ev2Lp4xZZDfgGPek7g35IsaiohH6YvHloxE+ETQ0Ttb1lCQos9fj7LlI+ud6ExjwyRuEnzCOzr/qC/3Leg85Wc8=
-    - secure: GbJVFFY4nmQovp8i2VoQ1AOV8h9QoAcleBV9Mg5TItA8U2Y+6t9wx0iWbaclxOrzI727XZOk5HquMTKKTI430lT6OjU01+YJqwyq+k650YiX5PImsgJXGhFzBKeUcB3A53kKijfr9aEV00YiXRsKoKAn+Bl09m/9O+melBbTwO/YAQyt6EoUt21Yk0BoqpW2CTMPAgsU5tm6Ra3bbJ+IaUYVMYBtuHqtGiKg8jgTHqRhwKejDyV7+1dDxX8q51bK2Kb6PujbI1SS5wsJEpkU/eoPs7MuprmlYarYtBt1fDXSK1EOjcfpyEFRlrNLewIKiFEo3dkTOEtW2a4t3tcAiXOj8Hcvp+6RsjTfnjYmZweHVh/JKDhujjHBMviyTjwp4RnPaQSFou0iVIB5+Mo34hUhFDXc30SBzaux+lyR3rGoEu5pubZNjG5CkKIpw3IyduNKtIrlbCnfg+WDU3jrXmk0fzmwyriyXhx9BAzM+hyFiElZ+3xoXAlmDlwvaRZyJuQQNslwGPbZyX9EByIUkECjYPf92QPyD0mjbn8C32IphgQ515Up+15g2meDniugzowLJkDe3W86M5fB1kPQiDq5oT4M6TgAPRElLN6/ibyfOzDAtOdZgUiQvFnUQb1s9LunZqrc57AKzuzl2xG9Ok/cUkZFSOjrFr40jXgS2Lw=

.travis/docker.sh

@@ -1,68 +0,0 @@
-#!/bin/bash -e
-
-login() {
-  echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-}
-
-tag_push() {
-  docker tag $DOCKER_USERNAME/zigbee2mqtt:$1 $DOCKER_USERNAME/zigbee2mqtt:$2
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$2
-}
-
-build_and_push() {
-  docker build --platform=$3 --build-arg COMMIT=$(git rev-parse --short HEAD) -t $DOCKER_USERNAME/zigbee2mqtt:$1 -f $2 .
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$1
-}
-
-push_manifest() {
-  export DOCKER_CLI_EXPERIMENTAL=enabled  
-
-  docker manifest create $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm32v6$1 $DOCKER_USERNAME/zigbee2mqtt:arm64v8$1
-
-  docker manifest annotate $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm32v6$1 --os linux --arch arm --variant v6
-  docker manifest annotate $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm64v8$1 --os linux --arch arm64 --variant v8
-
-  docker manifest inspect $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  docker manifest push -p $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  docker run --rm mplatform/mquery $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  export DOCKER_CLI_EXPERIMENTAL=disabled
-}
-
-# Only update Docker images for:
-# - dev branch
-# - version release
-if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ ! -z ${TRAVIS_TAG+x} ] && [ ! "$TRAVIS_TAG" = "" ]
-then
-  echo "Updating docker images for master branch!"
-  login
-
-  # Push versioned images
-  build_and_push "$TRAVIS_TAG" docker/Dockerfile.amd64 amd64
-  build_and_push "$TRAVIS_TAG-arm32v6" docker/Dockerfile.arm32v6 arm
-  build_and_push "$TRAVIS_TAG-arm64v8" docker/Dockerfile.arm64v8 arm64
-
-  # Push latest images.
-  tag_push "$TRAVIS_TAG-arm32v6" "arm32v6"
-  tag_push "$TRAVIS_TAG-arm64v8" "arm64v8"
-  tag_push "$TRAVIS_TAG" latest
-
-  # Push manifest
-  push_manifest ""
-
-elif [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Updating docker images for dev branch!"
-  login
-  build_and_push latest-dev docker/Dockerfile.amd64 amd64
-  build_and_push arm32v6-dev docker/Dockerfile.arm32v6 arm
-  build_and_push arm64v8-dev docker/Dockerfile.arm64v8 arm64
-
-  # Push manifest
-  push_manifest "-dev"
-
-else
-  echo "Not updating docker images, triggered by pull request or not on master/dev branch"
-fi

.travis/trigger-downstream.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-function travis_trigger() {
-     local org=$1 && shift
-     local repo=$1 && shift
-     local branch=${1:-master} && shift
-
-     body="{
-             \"request\": {
-               \"branch\": \"${branch}\"
-              }
-           }"
-
-     curl -s -X POST \
-          -H "Content-Type: application/json" \
-          -H "Accept: application/json" \
-          -H "Travis-API-Version: 3" \
-          -H "Authorization: token $TRAVIS_TOKEN" \
-          -d "$body" \
-          "https://api.travis-ci.org/repo/${org}%2F${repo}/requests"
-}
-
-# Only trigger downstream if on dev branch and not pull request
-if [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Triggering build of downstream projects!"
-  travis_trigger "danielwelch" "hassio-zigbee2mqtt" "master"
-else
-  echo "Not triggering build of downstream projects, triggered by pull request or not on dev branch"
-fi

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -0,0 +1,8 @@
+# Contributing to Zigbee2MQTT
+
+Everybody is invited and welcome to contribute to Zigbee2MQTT. Zigbee2MQTT is writen in JavaScript and is based upon [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) and [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters). Zigbee-herdsman-converters contains all device definition, zigbee-herdsman is responsible for handling all communication with the adapter.
+
+- Pull requests are always created against the [**dev**](https://github.com/Koenkk/zigbee2mqtt/tree/dev) branch.
+- Easiest way to start developing Zigbee2MQTT is by setting up a development environment (aka bare-metal installation). You can follow this [guide](https://www.zigbee2mqtt.io/getting_started/running_zigbee2mqtt.html#running-zigbee2mqtt) to do this.
+- You can run the tests locally by executing `npm test`. Zigbee2MQTT enforces 100% code coverage, in case you add new code check if you code is covered by running `npm run test-with-coverage`. The coverage report can be found under `coverage/lcov-report/index.html`. Linting is also enforced and can be run with `npm run eslint`.
+- When you want to add support for a new devices no changes to Zigbee2MQTT have to be made, only to zigbee-herdsman-converters. You can find a guide for it [here](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).

README.md

@@ -5,8 +5,8 @@
     <br>
     <br>
     <div style="display: flex;">
-        <a href="https://travis-ci.org/Koenkk/zigbee2mqtt/">
-            <img src="https://img.shields.io/travis/Koenkk/zigbee2mqtt.svg">
+        <a href="https://github.com/Koenkk/zigbee2mqtt/actions?query=workflow%3ACI">
+            <img src="https://github.com/koenkk/zigbee2mqtt/workflows/CI/badge.svg">
         </a>
         <a href="https://github.com/Koenkk/zigbee2mqtt/releases">
             <img src="https://img.shields.io/github/release/koenkk/zigbee2mqtt.svg">
@@ -23,8 +23,17 @@
         <a href="http://zigbee2mqtt.discourse.group/">
             <img src="https://img.shields.io/discourse/https/zigbee2mqtt.discourse.group/status.svg">
         </a>
+        <a>
+            <img src="https://img.shields.io/badge/Coverage-100%25-brightgreen.svg">
+        </a>
+        <a href="https://www.codacy.com/manual/Koenkk/zigbee2mqtt?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=Koenkk/zigbee2mqtt&amp;utm_campaign=Badge_Grade">
+            <img src="https://api.codacy.com/project/badge/Grade/24f1e0fe39f04daa810e8a1416693d3f">
+        </a>
+        <a href="https://www.npmjs.com/package/zigbee2mqtt">
+            <img src="https://img.shields.io/npm/v/zigbee2mqtt">
+        </a>
     </div>
-    <h1>Zigbee2mqtt  🌉 🐝</h1>
+    <h1>Zigbee2MQTT  🌉 🐝</h1>
     <p>
         Allows you to use your Zigbee devices <b>without</b> the vendors bridge or gateway.
     </p>
@@ -39,12 +48,12 @@ The [documentation](https://www.zigbee2mqtt.io/) provides you all the informatio
 If you aren't familiar with **Zigbee** terminology make sure you [read this](https://www.zigbee2mqtt.io/information/zigbee_network.html) to help you out.
 
 ## [Integrations](https://www.zigbee2mqtt.io/#integration)
-Zigbee2mqtt integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
+Zigbee2MQTT integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">
 
 ### [Home Assistant](https://www.home-assistant.io/)
-- [Hassio](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
+- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/zigbee2mqtt/hassio-zigbee2mqtt)
 - Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">
@@ -57,6 +66,12 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ## Architecture
 ![Architecture](images/architecture.png)
 
+### Internal Architecture
+Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
+### Developing
+Zigbee2MQTT uses TypeScript (partially for now). Therefore after making changes to files in the `lib/` directory you need to recompile Zigbee2MQTT. This can be done by executing `npm run build`. For faster development instead of running `npm run build` you can run `npm run build-watch` in another terminal session, this will recompile as you change files.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

babel.config.js

@@ -0,0 +1,11 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', {targets: {node: 'current'}}],
+        '@babel/preset-typescript',
+    ],
+    assumptions: {setPublicClassFields: true},
+    plugins: [
+        ['@babel/plugin-proposal-decorators', {'legacy': true}],
+        ['@babel/plugin-proposal-class-properties'],
+    ],
+};

cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+const path = require('path');
+process.env['ZIGBEE2MQTT_DATA'] = process.env['ZIGBEE2MQTT_DATA'] || path.join(process.env['HOME'], '.z2m');
+require('./index');

docker/Dockerfile

@@ -0,0 +1,37 @@
+FROM node:14-alpine3.12 as base
+
+WORKDIR /app
+RUN apk add --no-cache tzdata eudev tini
+
+COPY package.json ./
+
+# Dependencies and build
+FROM base as dependencies_and_build
+
+COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY lib ./lib
+
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
+    npm ci --no-audit --no-optional --no-update-notifier && \
+    npm run build && \
+    rm -rf node_modules && \
+    npm ci --production --no-audit --no-optional --no-update-notifier && \
+    apk del .buildtools
+
+# Release
+FROM base as release
+
+COPY --from=dependencies_and_build /app/node_modules ./node_modules
+COPY --from=dependencies_and_build /app/dist ./dist
+COPY LICENSE index.js data/configuration.yaml ./
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+
+RUN mkdir /app/data
+
+ARG COMMIT
+RUN echo "$COMMIT" > dist/.hash
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD [ "/sbin/tini", "--", "node", "index.js"]

docker/Dockerfile.amd64

@@ -1,20 +0,0 @@
-FROM alpine:3.9
-
-# Copy files
-ADD . /app
-RUN cp /app/data/configuration.yaml /app
-RUN cp /app/docker/run.sh /app
-RUN chmod +x /app/run.sh
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
-
-# Install dependencies
-RUN apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-    npm install --unsafe-perm && \
-    apk del make gcc g++ python linux-headers udev git
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm32v6

@@ -1,22 +0,0 @@
-FROM arm32v6/alpine:3.9
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-arm-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-arm-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-arm-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm64v8

@@ -1,22 +0,0 @@
-FROM arm64v8/alpine:3.9
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-aarch64-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/README.md

@@ -1 +0,0 @@
-qemu-aarch64-static and qemu-arm-static are taken from https://github.com/resin-io/qemu/releases/tag/v2.9.0%2Bresin1

docker/docker-entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 if [ ! -z "$ZIGBEE2MQTT_DATA" ]; then
     DATA="$ZIGBEE2MQTT_DATA"
@@ -13,4 +14,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-exec npm start
+exec "$@"

docs/_config.yml

@@ -1 +0,0 @@
-theme: jekyll-theme-cayman

docs/index.md

@@ -1 +0,0 @@
-Moved to [https://www.zigbee2mqtt.io](https://www.zigbee2mqtt.io)

index.js

@@ -1,23 +1,115 @@
 const semver = require('semver');
 const engines = require('./package.json').engines;
+const indexJsRestart = 'indexjs.restart';
+const fs = require('fs');
+const path = require('path');
+const {exec} = require('child_process');
+const rimraf = require('rimraf');
+require('source-map-support').install();
 
-const version = engines.node;
-if (!semver.satisfies(process.version, version)) {
-    console.log(`\t\tZigbee2mqtt requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
-}
-
-const Controller = require('./lib/controller');
-const controller = new Controller();
-controller.start();
-
-process.on('SIGINT', handleQuit);
-process.on('SIGTERM', handleQuit);
-
+let controller;
 let stopping = false;
 
-function handleQuit() {
-    if (!stopping) {
-        stopping = true;
-        controller.stop(() => process.exit());
+const hashFile = path.join(__dirname, 'dist', '.hash');
+
+async function restart() {
+    await stop(indexJsRestart);
+    await start();
+}
+
+async function exit(code, reason) {
+    if (reason !== indexJsRestart) {
+        process.exit(code);
     }
 }
+
+async function currentHash() {
+    const git = require('git-last-commit');
+    return new Promise((resolve) => {
+        git.getLastCommit((err, commit) => {
+            if (err) resolve('unknown');
+            else resolve(commit.shortHash);
+        });
+    });
+}
+
+async function writeHash() {
+    const hash = await currentHash();
+    fs.writeFileSync(hashFile, hash);
+}
+
+async function build(reason) {
+    return new Promise((resolve, reject) => {
+        process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
+        rimraf.sync('dist');
+        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+            if (err) {
+                process.stdout.write(', failed\n');
+                reject(err);
+            } else {
+                process.stdout.write(', finished\n');
+                resolve();
+            }
+        });
+    });
+}
+
+async function checkDist() {
+    if (!fs.existsSync(hashFile)) {
+        await build('initial build');
+    }
+
+    const distHash = fs.readFileSync(hashFile, 'utf-8');
+    const hash = await currentHash();
+    if (hash !== 'unknown' && distHash !== hash) {
+        await build('hash changed');
+    }
+}
+
+async function start() {
+    await checkDist();
+
+    const version = engines.node;
+    if (!semver.satisfies(process.version, version)) {
+        console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+    }
+
+    // Validate settings
+    const settings = require('./dist/util/settings');
+    settings.reRead();
+    const errors = settings.validate();
+    if (errors.length > 0) {
+        console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
+        console.log('            READ THIS CAREFULLY\n');
+        console.log(`Refusing to start because configuration is not valid, found the following errors:`);
+        for (const error of errors) {
+            console.log(`- ${error}`);
+        }
+        console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`); // eslint-disable-line
+        console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
+        exit(1);
+    }
+
+    const Controller = require('./dist/controller');
+    controller = new Controller(restart, exit);
+    await controller.start();
+}
+
+async function stop(reason=null) {
+    await controller.stop(reason);
+}
+
+async function handleQuit() {
+    if (!stopping && controller) {
+        stopping = true;
+        await stop();
+    }
+}
+
+if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+    writeHash();
+} else {
+    process.on('SIGINT', handleQuit);
+    process.on('SIGTERM', handleQuit);
+    start();
+}

lib/controller.js

@@ -1,102 +1,149 @@
 const MQTT = require('./mqtt');
 const Zigbee = require('./zigbee');
+const EventBus = require('./eventBus');
 const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const stringify = require('json-stable-stringify-without-jsonify');
+const assert = require('assert');
 
 // Extensions
+const ExtensionFrontend = require('./extension/frontend');
+const ExtensionPublish = require('./extension/publish');
+const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
-const ExtensionSoftReset = require('./extension/softReset');
-const ExtensionXiaomi = require('./extension/xiaomi');
-const ExtensionDevicePublish = require('./extension/devicePublish');
+const ExtensionSoftReset = require('./extension/legacy/softReset');
 const ExtensionHomeAssistant = require('./extension/homeassistant');
-const ExtensionDeviceConfigure = require('./extension/deviceConfigure');
-const ExtensionDeviceGroupMembership = require('./extension/deviceGroupMembership');
-const ExtensionDeviceReceive = require('./extension/deviceReceive');
-const ExtensionBridgeConfig = require('./extension/bridgeConfig');
+const ExtensionConfigure = require('./extension/configure');
+const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
+const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
-const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
-const ExtensionDeviceBind = require('./extension/deviceBind');
-const ExtensionDeviceReport = require('./extension/deviceReport');
-const ExtensionLivolo = require('./extension/livolo');
-const ExtensionResponder = require('./extension/responder');
+const ExtensionAvailabilityLegacy = require('./extension/legacy/availability');
+const ExtensionAvailability = require('./extension/availability');
+const ExtensionBind = require('./extension/bind');
+const ExtensionReport = require('./extension/legacy/report');
+const ExtensionOnEvent = require('./extension/onEvent');
+const ExtensionOTAUpdate = require('./extension/otaUpdate');
+const ExtensionExternalConverters = require('./extension/externalConverters');
+const ExtensionExternalExtension = require('./extension/externalExtension');
+
+const AllExtensions = [
+    ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
+    ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
+    ExtensionAvailabilityLegacy, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension, ExtensionAvailability,
+];
 
 class Controller {
-    constructor() {
-        this.zigbee = new Zigbee();
-        this.mqtt = new MQTT();
-        this.state = new State();
+    constructor(restartCallback, exitCallback) {
+        // eslint-disable-next-line
+        this.eventBus = new EventBus.default();
+        // eslint-disable-next-line
+        this.zigbee = new Zigbee.default(this.eventBus);
+        // eslint-disable-next-line
+        this.mqtt = new MQTT.default(this.eventBus);
+        this.state = new State(this.eventBus);
+        this.restartCallback = restartCallback;
+        this.exitCallback = exitCallback;
 
-        // Bind methods
-        this.onMQTTConnected = this.onMQTTConnected.bind(this);
-        this.onZigbeeMessage = this.onZigbeeMessage.bind(this);
-        this.onMQTTMessage = this.onMQTTMessage.bind(this);
         this.publishEntityState = this.publishEntityState.bind(this);
+        this.enableDisableExtension = this.enableDisableExtension.bind(this);
+        this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
+        this.addExtension = this.addExtension.bind(this);
 
         // Initialize extensions.
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+            this.enableDisableExtension, this.restartCallback, this.addExtension];
         this.extensions = [
-            new ExtensionDeviceReceive(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceGroupMembership(this.zigbee, this.mqtt, this.publishEntityState),
-            new ExtensionDeviceConfigure(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDevicePublish(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionNetworkMap(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionXiaomi(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionBridgeConfig(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionGroups(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceBind(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionResponder(this.zigbee, this.mqtt, this.state, this.publishEntityState),
+            new ExtensionBridge(...args),
+            new ExtensionPublish(...args),
+            new ExtensionReceive(...args),
+            new ExtensionDeviceGroupMembership(...args),
+            new ExtensionConfigure(...args),
+            new ExtensionNetworkMap(...args),
+            new ExtensionGroups(...args),
+            new ExtensionBind(...args),
+            new ExtensionOnEvent(...args),
+            new ExtensionOTAUpdate(...args),
+            new ExtensionReport(...args),
         ];
 
-        if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionDeviceReport(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+        if (settings.get().frontend) {
+            this.extensions.push(new ExtensionFrontend(...args));
+        }
+
+        if (settings.get().advanced.legacy_api) {
+            this.extensions.push(new ExtensionBridgeLegacy(...args));
+        }
+
+        if (settings.get().external_converters.length) {
+            this.extensions.push(new ExtensionExternalConverters(...args));
         }
 
         if (settings.get().homeassistant) {
-            this.extensions.push(new ExtensionHomeAssistant(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionHomeAssistant(...args));
         }
 
+        /* istanbul ignore next */
         if (settings.get().advanced.soft_reset_timeout !== 0) {
-            this.extensions.push(new ExtensionSoftReset(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionSoftReset(...args));
         }
 
-        if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionDeviceAvailability(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+        /* istanbul ignore else */
+        if (settings.get().experimental.availability_new) {
+            this.extensions.push(new ExtensionAvailability(...args));
+        } else {
+            if (settings.get().advanced.availability_timeout) {
+                this.extensions.push(new ExtensionAvailabilityLegacy(...args));
+            }
         }
 
-        if (settings.get().experimental.livolo) {
-            // https://github.com/Koenkk/zigbee2mqtt/issues/592
-            this.extensions.push(new ExtensionLivolo(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
-        }
+        this.extensions.push(new ExtensionExternalExtension(...args));
     }
 
-    onMQTTConnected() {
-        // Resend all cached states.
-        this.sendAllCachedStates();
+    async start() {
+        this.state.start();
+        logger.logOutput();
 
-        // Call extensions
-        this.extensions.filter((e) => e.onMQTTConnected).forEach((e) => e.onMQTTConnected());
-    }
+        const info = await utils.getZigbee2MQTTVersion();
+        logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
-    onZigbeeStarted() {
-        // Log zigbee clients on startup and configure.
-        const devices = this.zigbee.getAllClients();
+        // Start zigbee
+        let startResult;
+        try {
+            startResult = await this.zigbee.start();
+            this.callExtensionMethod('onZigbeeStarted', []);
+            this.eventBus.on('event', this.onZigbeeEvent.bind(this), this.constructor.name);
+            this.eventBus.onAdapterDisconnected(this, this.onZigbeeAdapterDisconnected);
+        } catch (error) {
+            logger.error('Failed to start zigbee');
+            // eslint-disable-next-line
+            logger.error('Check https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start for possible solutions');
+            logger.error('Exiting...');
+            logger.error(error.stack);
+            this.exitCallback(1);
+        }
+
+        // Disable some legacy options on new network creation
+        if (startResult === 'reset') {
+            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['device_options', 'legacy'], false);
+            this.enableDisableExtension(false, 'BridgeLegacy');
+        }
+
+        // Log zigbee clients on startup
+        const devices = this.zigbee.getClientsLegacy();
         logger.info(`Currently ${devices.length} devices are joined:`);
-        devices.forEach((device) => {
-            logger.info(this.getDeviceStartupLogMessage(device));
-        });
+        for (const device of devices) {
+            const entity = this.zigbee.resolveEntityLegacy(device);
+            const model = entity.definition ?
+                `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description}` :
+                'Not supported';
+            logger.info(`${entity.name} (${entity.device.ieeeAddr}): ${model} (${entity.device.type})`);
+        }
 
         // Enable zigbee join.
         if (settings.get().permit_join) {
@@ -105,206 +152,290 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        this.zigbee.permitJoin(settings.get().permit_join);
-
-        // Connect to MQTT broker
-        this.mqtt.connect(this.onMQTTMessage, this.onMQTTConnected);
-
-        // Call extensions
-        this.extensions.filter((e) => e.onZigbeeStarted).forEach((e) => e.onZigbeeStarted());
-    }
-
-    onZigbeeMessage(message) {
-        // Variables
-        let device = null;
-        let mappedDevice = null;
-
-        // Check if message has a device
-        if (message.endpoints && message.endpoints.length && message.endpoints[0].device) {
-            device = message.endpoints[0].device;
+        try {
+            await this.zigbee.permitJoinLegacy(settings.get().permit_join);
+        } catch (error) {
+            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
         }
 
-        // Retrieve modelId from message
-        if (device && device.modelId) {
-            mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        }
+        // MQTT
+        this.onMQTTMessage = this.onMQTTMessage.bind(this);
+        this.eventBus.onMQTTMessage(this.constructor.name, this.onMQTTMessage);
+        await this.mqtt.connect();
 
-        // Log
-        logger.debug(
-            `Received zigbee message of type '${message.type}' with data '${JSON.stringify(message.data)}'` +
-            (device ? ` of device '${device.modelId}' (${device.ieeeAddr})` : '') +
-            (message.groupid ? ` with groupID ${message.groupid}` : '') +
-            (message.endpoints && message.endpoints.length ? ` of endpoint ${message.endpoints[0].epId}` : '')
-        );
-
-        // Call extensions.
-        this.extensions
-            .filter((e) => e.onZigbeeMessage)
-            .forEach((e) => e.onZigbeeMessage(message, device, mappedDevice));
-    }
-
-    onMQTTMessage(topic, message) {
-        logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
-
-        // Call extensions
-        const results = this.extensions
-            .filter((e) => e.onMQTTMessage)
-            .map((e) => e.onMQTTMessage(topic, message));
-
-        if (!results.includes(true)) {
-            logger.warn(`Cannot handle MQTT message on '${topic}' with data '${message}'`);
-        }
-    }
-
-    start() {
-        logger.info(`Logging to directory: '${logger.directory}'`);
-
-        logger.cleanup();
-
-        this.state.start();
-
-        this.startupLogVersion(() => {
-            this.zigbee.start(this.onZigbeeMessage, (error) => {
-                if (error) {
-                    logger.error('Failed to start', error);
-                    logger.error('Exiting...');
-                    process.exit(1);
-                } else {
-                    this.onZigbeeStarted();
+        // Send all cached states.
+        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
+            for (const device of this.zigbee.getClientsLegacy()) {
+                if (this.state.exists(device.ieeeAddr)) {
+                    this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
                 }
-            });
-        });
+            }
+        }
+
+        // Add devices which are in database but not in configuration to configuration
+        for (const device of this.zigbee.getClientsLegacy()) {
+            if (!settings.getDevice(device.ieeeAddr)) {
+                settings.addDevice(device.ieeeAddr);
+            }
+        }
+
+        // Call extensions
+        await this.callExtensionMethod('onMQTTConnected', []);
+        await this.callExtensionMethod('start', []);
     }
 
-    stop(callback) {
+    async enableDisableExtension(enable, name) {
+        if (!enable) {
+            const extension = this.extensions.find((e) => e.constructor.name === name);
+            if (extension) {
+                await this.callExtensionMethod('stop', [], [extension]);
+                this.extensions.splice(this.extensions.indexOf(extension), 1);
+            }
+        } else {
+            const Extension = AllExtensions.find((e) => e.name === name);
+            assert(Extension, `Extension '${name}' does not exist`);
+            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+                this.enableDisableExtension, this.restartCallback, this.addExtension);
+
+            this.extensions.push(extension);
+            await this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+            await this.callExtensionMethod('onMQTTConnected', [], [extension]);
+            await this.callExtensionMethod('start', [], [extension]);
+        }
+    }
+
+    async addExtension(extension) {
+        this.extensions.push(extension);
+        await this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+        await this.callExtensionMethod('onMQTTConnected', [], [extension]);
+        await this.callExtensionMethod('start', [], [extension]);
+    }
+
+    async stop(reason=null) {
         // Call extensions
-        this.extensions.filter((e) => e.stop).forEach((e) => e.stop());
+        await this.callExtensionMethod('stop', []);
+        this.eventBus.removeListeners(this.constructor.name);
 
         // Wrap-up
         this.state.stop();
+        await this.mqtt.disconnect();
 
-        this.mqtt.disconnect();
-        this.zigbee.stop(callback);
+        try {
+            await this.zigbee.stop();
+            logger.info('Stopped Zigbee2MQTT');
+            this.exitCallback(0, reason);
+        } catch (error) {
+            logger.error('Failed to stop Zigbee2MQTT');
+            this.exitCallback(1, reason);
+        }
     }
 
-    startupLogVersion(callback) {
-        utils.getZigbee2mqttVersion((info) => {
-            logger.info(`Starting zigbee2mqtt version ${info.version} (commit #${info.commitHash})`);
-            callback();
-        });
+    async onZigbeeAdapterDisconnected() {
+        logger.error('Adapter disconnected, stopping');
+        await this.stop();
     }
 
-    getDeviceStartupLogMessage(device) {
-        let friendlyName = 'unknown';
-        let type = 'unknown';
-        let friendlyDevice = {model: 'unkown', description: 'unknown'};
-        const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        if (mappedModel) {
-            friendlyDevice = mappedModel;
+    async onZigbeeEvent(type, data) {
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(data.device || data.ieeeAddr);
+        if (data.device && data.device.type === 'Coordinator') {
+            logger.debug('Ignoring message from coordinator');
+            return;
         }
 
-        if (settings.getDevice(device.ieeeAddr)) {
-            friendlyName = settings.getDevice(device.ieeeAddr).friendly_name;
+        if (data.device && !settings.getDevice(data.device.ieeeAddr)) {
+            // Only deviceLeave doesn't have a device (not interesting to add to settings)
+            resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
         }
 
-        if (device.type) {
-            type = utils.correctDeviceType(device);
-        }
+        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
 
-        return `${friendlyName} (${device.ieeeAddr}): ${friendlyDevice.model} - ` +
-            `${friendlyDevice.vendor} ${friendlyDevice.description} (${type})`;
-    }
+        if (type === 'message') {
+            logger.debug(
+                `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
+                `, data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
+            );
+        } else if (type === 'deviceJoined') {
+            logger.info(`Device '${name}' joined`);
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
 
-    sendAllCachedStates() {
-        this.zigbee.getAllClients().forEach((device) => {
-            if (this.state.exists(device.ieeeAddr)) {
-                this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
+                } else {
+                    logger.warn(
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' and manufacturer name ` +
+                        `'${data.device.manufacturerName}' is NOT supported, ` +
+                        `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
+                    );
+                }
+            } else if (data.status === 'failed') {
+                logger.error(`Failed to interview '${name}', device has not successfully been paired`);
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    logger.info(`Starting interview of '${name}'`);
+                }
             }
-        });
+        } else if (type === 'deviceAnnounce') {
+            logger.debug(`Device '${name}' announced itself`);
+        } else if (type === 'deviceNetworkAddressChanged') {
+            logger.debug(`Device '${name}' changed network address`);
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const settingsDevice = settings.getDevice(data.ieeeAddr);
+                logger.warn(`Device '${settingsDevice ? settingsDevice.friendlyName : data.ieeeAddr}' ` +
+                    `left the network`);
+                this.state.remove(data.ieeeAddr);
+            }
+        }
+
+        // Call extensions
+        await this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+
+        // In case this message is not handled by the receive extension and last_seen is enabled, publish
+        // a message to update the last_seen.
+        // TODO
+        // if (name && !result[ExtensionReceive.prototype.constructor.name] &&
+        //     settings.get().advanced.last_seen !== 'disable' && type !== 'deviceLeave') {
+        //     this.publishEntityState(name, {});
+        // }
     }
 
-    publishEntityState(entityID, payload) {
-        const entity = settings.resolveEntity(entityID);
-        const appSettings = settings.get();
+    onMQTTMessage(payload) {
+        const {topic, message} = payload;
+        logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
+
+        // Call extensions
+        this.callExtensionMethod('onMQTTMessage', [topic, message]);
+    }
+
+    async publishEntityState(IDorName, payload, stateChangeReason=null) {
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(IDorName);
+        if (!resolvedEntity || !resolvedEntity.settings) {
+            logger.error(`'${IDorName}' does not exist, skipping publish`);
+            return;
+        }
+
         let messagePayload = {...payload};
 
-        const currentState = this.state.exists(entityID) ? this.state.get(entityID) : {};
-        const newState = objectAssignDeep.noMutate(currentState, payload);
-
         // Update state cache with new state.
-        this.state.set(entityID, newState);
+        const newState = this.state.set(resolvedEntity.settings.ID, payload, stateChangeReason);
 
         if (settings.get().advanced.cache_state) {
             // Add cached state to payload
             messagePayload = newState;
         }
 
-        const entitySettings = entity.type === 'device' ? settings.getDevice(entityID) : settings.getGroup(entityID);
         const options = {
-            retain: entitySettings ? entitySettings.retain : false,
-            qos: entitySettings && entitySettings.qos ? entitySettings.qos : 0,
+            retain: utils.getObjectProperty(resolvedEntity.settings, 'retain', false),
+            qos: utils.getObjectProperty(resolvedEntity.settings, 'qos', 0),
         };
 
-        if (entity.type === 'device' && appSettings.mqtt.include_device_information) {
-            messagePayload.device = this.getDeviceInfoForMqtt(entityID);
+        const retention = utils.getObjectProperty(resolvedEntity.settings, 'retention', false);
+        if (retention !== false) {
+            options.properties = {messageExpiryInterval: retention};
+        }
+
+        const isDevice = resolvedEntity.type === 'device';
+        if (isDevice && settings.get().mqtt.include_device_information) {
+            const attributes = [
+                'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
+                'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
+            ];
+
+            messagePayload.device = {
+                friendlyName: resolvedEntity.name,
+                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
+            };
+
+            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
+        }
+
+        // Add lastseen
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+        }
+
+        // Add device linkquality.
+        if (resolvedEntity.type === 'device' && resolvedEntity.device.linkquality !== undefined) {
+            messagePayload.linkquality = resolvedEntity.device.linkquality;
+        }
+
+        // filter mqtt message attributes
+        if (resolvedEntity.settings.filtered_attributes) {
+            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        }
+
+        for (const extension of this.extensions) {
+            if (extension.adjustMessagePayloadBeforePublish) {
+                extension.adjustMessagePayloadBeforePublish(resolvedEntity, messagePayload);
+            }
         }
 
         if (Object.entries(messagePayload).length) {
-            if (settings.get().experimental.output === 'json') {
-                this.mqtt.publish(entity.friendlyName, JSON.stringify(messagePayload), options);
-            } else if (settings.get().experimental.output === 'attribute') {
-                this.iteratePayloadForAttrOutput(entity.friendlyName+'/', messagePayload, options);
+            const output = settings.get().experimental.output;
+            if (output === 'attribute_and_json' || output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, stringify(messagePayload), options);
+            }
+
+            if (output === 'attribute_and_json' || output === 'attribute') {
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
             }
         }
+
+        this.eventBus.emit('publishEntityState', {messagePayload, entity: resolvedEntity, stateChangeReason, payload,
+            ieeeAddr: resolvedEntity.type === 'device' && resolvedEntity.device.ieeeAddr});
     }
 
-    iteratePayloadForAttrOutput(topicRoot, payload, options) {
-        Object.keys(payload).forEach((key) => {
-            let subPayload = payload[key];
-            let message;
+    async iteratePayloadAttributeOutput(topicRoot, payload, options) {
+        for (const [key, value] of Object.entries(payload)) {
+            let subPayload = value;
+            let message = null;
 
             // Special cases
-            if (key === 'color' &&
-                subPayload.r !== undefined &&
-                subPayload.g !== undefined &&
-                subPayload.b !== undefined) {
+            if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
                 subPayload = [subPayload.r, subPayload.g, subPayload.b];
             }
 
             // Check Array first, since it is also an Object
-            if (Array.isArray(subPayload)) {
+            if (subPayload === null || subPayload === undefined) {
+                message = '';
+            } else if (Array.isArray(subPayload)) {
                 message = subPayload.map((x) => `${x}`).join(',');
             } else if (typeof subPayload === 'object') {
-                return this.iteratePayloadForAttrOutput(topicRoot+key+'-', subPayload, options);
+                this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
             } else {
-                message = typeof subPayload === 'string' ? subPayload : JSON.stringify(subPayload);
+                message = typeof subPayload === 'string' ? subPayload : stringify(subPayload);
             }
 
-            this.mqtt.publish(`${topicRoot}${key}`, message, options);
-        });
+            if (message !== null) {
+                await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+            }
+        }
     }
 
-    getDeviceInfoForMqtt(ieeeAddr) {
-        const device = this.zigbee.getDevice(ieeeAddr);
-        const {
-            type, nwkAddr, manufId, manufName, powerSource, modelId, hwVersion, swBuildId, status, dateCode,
-        } = device;
-        const deviceSettings = settings.getDevice(device.ieeeAddr);
-
-        return {
-            ieeeAddr,
-            friendlyName: deviceSettings ? (deviceSettings.friendly_name || '') : '',
-            type,
-            nwkAddr,
-            manufId,
-            manufName,
-            powerSource,
-            modelId,
-            hwVersion: hwVersion ? hwVersion : 'unknown',
-            swBuildId: swBuildId ? swBuildId : 'unknown',
-            dateCode: dateCode ? dateCode : 'unknown',
-            status,
-        };
+    async callExtensionMethod(method, parameters, extensions=null) {
+        const result = {};
+        for (const extension of extensions || this.extensions) {
+            if (extension[method]) {
+                try {
+                    result[extension.constructor.name] = await extension[method](...parameters);
+                } catch (error) {
+                    /* istanbul ignore next */
+                    logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
+                    /* istanbul ignore next */
+                    if (process.env.JEST_WORKER_ID !== undefined) {
+                        throw error;
+                    }
+                }
+            }
+        }
+        return result;
     }
 }
 

lib/eventBus.ts

@@ -0,0 +1,139 @@
+/* eslint-disable brace-style */
+import events from 'events';
+import * as ZHEvents from 'zigbee-herdsman/dist/controller/events';
+import ExtensionTS from './extension/extensionts';
+
+declare global {
+    interface EventDeviceRenamed { device: Device, homeAssisantRename: boolean, from: string, to: string }
+    // TODO: remove resolved entity, replace by Device
+    interface EventDeviceRemoved { resolvedEntity: ResolvedEntity}
+    type EventMQTTMessage = { topic: string, message: string };
+    type EventMQTTMessagePublished = { topic: string, payload: string, options: {retain: boolean, qos: number} };
+    type EventPermitJoinChanged = ZHEvents.PermitJoinChangedPayload;
+    type EventLastSeenChanged = { device: Device };
+    type EventDeviceNetworkAddressChanged = { device: Device };
+    type EventDeviceAnnounce = { device: Device };
+    type EventDeviceInterview = { device: Device, status: 'started' | 'successful' | 'failed' };
+    type EventDeviceJoined = { device: Device };
+    type EventReportingDisabled = { device: ZHDevice }; // TODO zhdevice -> device
+    type EventDeviceLeave = { ieeeAddr: string };
+    // TODO: remove resolved entity, replace by Group
+    type EventGroupMembersChanged = { group: ResolvedEntity }; // TODO fill
+    type EventPublishEntityState = {
+        // TODO: remove resolved entity, replace by Device | Group and remove ieeeAddr
+        messagePayload: KeyValue, entity: ResolvedEntity, stateChangeReason: 'publishDebounce', payload: KeyValue,
+        ieeeAddr: string,
+    };
+    type EventDeviceMessage = {
+        type: ZHEvents.MessagePayloadType;
+        device: Device;
+        endpoint: ZHEndpoint;
+        linkquality: number;
+        groupID: number;
+        cluster: string | number;
+        data: KeyValue | Array<string | number>;
+        meta: {
+            zclTransactionSequenceNumber?: number;
+        };
+    };
+}
+
+type ListenerKey = string | ExtensionTS;
+
+export default class EventBus {
+    /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+    private callbacksByExtension: { [s: string]: { event: string, callback: (...args: any[]) => void }[] } = {};
+    private emitter = new events.EventEmitter();
+
+    public emitAdapterDisconnected(): void {this.emitter.emit('adapterDisconnected');}
+    public onAdapterDisconnected(key: ListenerKey, callback: () => void): void {
+        this.on('adapterDisconnected', callback, key);}
+
+    public emitPermitJoinChanged(data: EventPermitJoinChanged): void {this.emitter.emit('permitJoinChanged', data);}
+    public onPermitJoinChanged(key: ListenerKey, callback: (data: EventPermitJoinChanged) => void): void {
+        this.on('permitJoinChanged', callback, key);}
+
+    // public emitDeviceRenamed(data: EventDeviceRenamed): void {this.emitter.emit('deviceRenamed', data);}
+    public onDeviceRenamed(key: ListenerKey, callback: (data: EventDeviceRenamed) => void): void {
+        this.on('deviceRenamed', callback, key);}
+
+    // public emitDeviceRemoved(data: EventDeviceRemoved): void {this.emitter.emit('deviceRemoved', data);}
+    public onDeviceRemoved(key: ListenerKey, callback: (data: EventDeviceRemoved) => void): void {
+        this.on('deviceRemoved', callback, key);}
+
+    public emitLastSeenChanged(data: EventLastSeenChanged): void {this.emitter.emit('lastSeenChanged', data);}
+    public onLastSeenChanged(key: ListenerKey, callback: (data: EventLastSeenChanged) => void): void {
+        this.on('lastSeenChanged', callback, key);}
+
+    public emitDeviceNetworkAddressChanged(data: EventDeviceNetworkAddressChanged): void {
+        this.emitter.emit('deviceNetworkAddressChanged', data);}
+    public onDeviceNetworkAddressChanged(
+        key: ListenerKey, callback: (data: EventDeviceNetworkAddressChanged) => void): void {
+        this.on('deviceNetworkAddressChanged', callback, key);}
+
+    public emitDeviceAnnounce(data: EventDeviceAnnounce): void {this.emitter.emit('deviceAnnounce', data);}
+    public onDeviceAnnounce(key: ListenerKey, callback: (data: EventDeviceAnnounce) => void): void {
+        this.on('deviceAnnounce', callback, key);}
+
+    public emitDeviceInterview(data: EventDeviceInterview): void {this.emitter.emit('deviceInterview', data);}
+    public onDeviceInterview(key: ListenerKey, callback: (data: EventDeviceInterview) => void): void {
+        this.on('deviceInterview', callback, key);}
+
+    public emitDeviceJoined(data: EventDeviceJoined): void {this.emitter.emit('deviceJoined', data);}
+    public onDeviceJoined(key: ListenerKey, callback: (data: EventDeviceJoined) => void): void {
+        this.on('deviceJoined', callback, key);}
+
+    public emitDeviceLeave(data: EventDeviceLeave): void {this.emitter.emit('deviceLeave', data);}
+    public onDeviceLeave(key: ListenerKey, callback: (data: EventDeviceLeave) => void): void {
+        this.on('deviceLeave', callback, key);}
+
+    public emitDeviceMessage(data: EventDeviceMessage): void {this.emitter.emit('deviceMessage', data);}
+    public onDeviceMessage(key: ListenerKey, callback: (data: EventDeviceMessage) => void): void {
+        this.on('deviceMessage', callback, key);}
+
+    public emitMQTTMessage(data: EventMQTTMessage): void {this.emitter.emit('mqttMessage', data);}
+    public onMQTTMessage(key: ListenerKey, callback: (data: EventMQTTMessage) => void): void {
+        this.on('mqttMessage', callback, key);}
+
+    public emitMQTTMessagePublished(data: EventMQTTMessagePublished): void {
+        this.emitter.emit('mqttMessagePublished', data);}
+    public onMQTTMessagePublished(key: ListenerKey, callback: (data: EventMQTTMessagePublished) => void): void {
+        this.on('mqttMessagePublished', callback, key);}
+
+    // public emitPublishEntityState(data: EventPublishEntityState): void {
+    //     this.emitter.emit('publishEntityState', data);}
+    public onPublishEntityState(key: ListenerKey, callback: (data: EventPublishEntityState) => void): void {
+        this.on('publishEntityState', callback, key);}
+
+    // public emitGroupMembersChanged(data: EventGroupMembersChanged): void {
+    //     this.emitter.emit('groupMembersChanged', data);}
+    public onGroupMembersChanged(key: ListenerKey, callback: (data: EventGroupMembersChanged) => void): void {
+        this.on('groupMembersChanged', callback, key);}
+
+    public emitDevicesChanged(): void {this.emitter.emit('devicesChanged');}
+    public onDevicesChanged(key: ListenerKey, callback: () => void): void {this.on('devicesChanged', callback, key);}
+
+    // public emitReportingDisabled(data: EventReportingDisabled): void {
+    //     this.emitter.emit('reportingDisabled', data);}
+    public onReportingDisabled(key: ListenerKey, callback: (data: EventReportingDisabled) => void): void {
+        this.on('reportingDisabled', callback, key);}
+
+    /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+    private on(event: string, callback: (...args: any[]) => void, key: ListenerKey): void {
+        key = typeof key === 'string' ? key : key.constructor.name;
+        if (!this.callbacksByExtension[key]) this.callbacksByExtension[key] = [];
+        this.callbacksByExtension[key].push({event, callback});
+        this.emitter.on(event, callback);
+    }
+
+    // TODO: remove
+    /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+    emit(event: string, ...args: any[]): void {
+        this.emitter.emit(event, ...args);
+    }
+
+    public removeListeners(key: ListenerKey): void {
+        key = typeof key === 'string' ? key : key.constructor.name;
+        this.callbacksByExtension[key]?.forEach((e) => this.emitter.removeListener(e.event, e.callback));
+    }
+}

lib/extension/availability.ts

@@ -0,0 +1,194 @@
+import ExtensionTS from './extensionts';
+import logger from '../util/logger';
+import {sleep, hours, minutes, seconds, isAvailabilityEnabledForDevice} from '../util/utils';
+import * as settings from '../util/settings';
+import debounce from 'debounce';
+import bind from 'bind-decorator';
+
+// TODO
+// - Enable for HA addon
+// - Add to setting schema (when old availability is removed)
+export default class Availability extends ExtensionTS {
+    private timers: {[s: string]: NodeJS.Timeout} = {};
+    private availabilityCache: {[s: string]: boolean} = {};
+    private retrieveStateDebouncers: {[s: string]: () => void} = {};
+    private pingQueue: Device[] = [];
+    private pingQueueExecuting = false;
+
+    private getTimeout(device: Device): number {
+        if (typeof device.settings.availability === 'object' && device.settings.availability?.timeout != null) {
+            return minutes(device.settings.availability.timeout);
+        }
+
+        const key = this.isActiveDevice(device) ? 'active' : 'passive';
+        const availabilitySettings = settings.get().availability;
+        if (typeof availabilitySettings === 'object' && availabilitySettings[key]?.timeout != null) {
+            return minutes(availabilitySettings[key]?.timeout);
+        }
+
+        return key === 'active' ? minutes(10) : hours(25);
+    }
+
+    private isActiveDevice(device: Device): boolean {
+        return (device.type === 'Router' && device.powerSource !== 'Battery') ||
+            device.powerSource === 'Mains (single phase)';
+    }
+
+    private isAvailable(device: Device): boolean {
+        const ago = Date.now() - device.lastSeen;
+        return ago < this.getTimeout(device);
+    }
+
+    private resetTimer(device: Device): void {
+        clearTimeout(this.timers[device.ieeeAddr]);
+
+        // If the timer triggers, the device is not avaiable anymore otherwise resetTimer already have been called
+        if (this.isActiveDevice(device)) {
+            // If device did not check in, ping it, if that fails it will be marked as offline
+            this.timers[device.ieeeAddr] = setTimeout(
+                () => this.addToPingQueue(device), this.getTimeout(device) + seconds(1));
+        } else {
+            this.timers[device.ieeeAddr] = setTimeout(
+                () => this.publishAvailability(device, true), this.getTimeout(device) + seconds(1));
+        }
+    }
+
+    private addToPingQueue(device: Device): void {
+        this.pingQueue.push(device);
+        this.pingQueueExecuteNext();
+    }
+
+    private removeFromPingQueue(device: Device): void {
+        const index = this.pingQueue.findIndex((d) => d.ieeeAddr === device.ieeeAddr);
+        index != -1 && this.pingQueue.splice(index, 1);
+    }
+
+    private async pingQueueExecuteNext(): Promise<void> {
+        if (this.pingQueue.length === 0 || this.pingQueueExecuting) return;
+        this.pingQueueExecuting = true;
+
+        const device = this.pingQueue[0];
+        let pingedSuccessfully = false;
+        const available = this.availabilityCache[device.ieeeAddr] || this.isAvailable(device);
+        const attempts = available ? 2 : 1;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                // Enable recovery if device is marked as available and first ping fails.
+                const disableRecovery = !(i == 1 && available);
+                await device.ping(disableRecovery);
+                pingedSuccessfully = true;
+                logger.debug(`Succesfully pinged '${device.name}' (attempt ${i + 1}/${attempts})`);
+                break;
+            } catch (error) {
+                logger.error(`Failed to ping '${device.name}' (attempt ${i + 1}/${attempts}, ${error.message})`);
+                // Try again in 3 seconds.
+                const lastAttempt = i - 1 === attempts;
+                !lastAttempt && await sleep(3);
+            }
+        }
+
+        this.publishAvailability(device, !pingedSuccessfully);
+        this.resetTimer(device);
+        this.removeFromPingQueue(device);
+
+        // Sleep 2 seconds before executing next ping
+        await sleep(2);
+        this.pingQueueExecuting = false;
+        this.pingQueueExecuteNext();
+    }
+
+    override async start(): Promise<void> {
+        logger.warn('Using experimental new availability feature');
+
+        this.eventBus.onDeviceRenamed(this, (data: EventDeviceRenamed) =>
+            this.publishAvailability(data.device, false, true));
+        this.eventBus.onDeviceLeave(this, (data: EventDeviceLeave) => clearTimeout(this.timers[data.ieeeAddr]));
+        this.eventBus.onDeviceAnnounce(this, (data: EventDeviceAnnounce) => this.retrieveState(data.device));
+        this.eventBus.onLastSeenChanged(this, this.onLastSeenChanged);
+
+        for (const device of this.zigbee.getClients()) {
+            if (isAvailabilityEnabledForDevice(device, settings.get())) {
+                // Publish initial availablility
+                this.publishAvailability(device, true);
+
+                this.resetTimer(device);
+
+                // If an active device is initially unavailable, ping it.
+                if (this.isActiveDevice(device) && !this.isAvailable(device)) {
+                    this.addToPingQueue(device);
+                }
+            }
+        }
+    }
+
+    private publishAvailability(device: Device, logLastSeen: boolean, forcePublish=false): void {
+        if (logLastSeen) {
+            const ago = Date.now() - device.lastSeen;
+            if (this.isActiveDevice(device)) {
+                logger.debug(
+                    `Active device '${device.name}' was last seen '${(ago / minutes(1)).toFixed(2)}' minutes ago.`);
+            } else {
+                logger.debug(
+                    `Passive device '${device.name}' was last seen '${(ago / hours(1)).toFixed(2)}' hours ago.`);
+            }
+        }
+
+        const available = this.isAvailable(device);
+        if (!forcePublish && this.availabilityCache[device.ieeeAddr] == available) {
+            return;
+        }
+
+        if (device.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[device.ieeeAddr] === false) {
+            logger.debug(`Device '${device.name}' reconnected`);
+            this.retrieveState(device);
+        }
+
+        const topic = `${device.name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        this.availabilityCache[device.ieeeAddr] = available;
+        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+    }
+
+    @bind private onLastSeenChanged(data: EventLastSeenChanged): void {
+        if (isAvailabilityEnabledForDevice(data.device, settings.get())) {
+            // Remove from ping queue, not necessary anymore since we know the device is online.
+            this.removeFromPingQueue(data.device);
+            this.resetTimer(data.device);
+            this.publishAvailability(data.device, false);
+        }
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        Object.values(this.timers).forEach((t) => clearTimeout(t));
+    }
+
+    private retrieveState(device: Device): void {
+        /**
+         * Retrieve state of a device in a debounced manner, this function is called on a 'deviceAnnounce' which a
+         * device can send multiple times after each other.
+         */
+        if (device.definition && !device.interviewing && !this.retrieveStateDebouncers[device.ieeeAddr]) {
+            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(async () => {
+                try {
+                    logger.debug(`Retrieving state of '${device.name}' after reconnect`);
+                    // Color and color temperature converters do both, only needs to be called once.
+                    const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
+                    for (const keys of keySet) {
+                        const converter = device.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
+                        await converter?.convertGet?.(device.endpoint(), keys[0],
+                            {message: this.state.get(device.ieeeAddr) || {}, mapped: device.definition});
+                    }
+                } catch (error) {
+                    logger.error(`Failed to read state of '${device.name}' after reconnect (${error.message})`);
+                }
+            }, seconds(2));
+        }
+
+        this.retrieveStateDebouncers[device.ieeeAddr]?.();
+    }
+}
+
+// TODO_finished: : change class to export default
+module.exports = Availability;

lib/extension/bind.js

@@ -0,0 +1,466 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
+const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+const debounce = require('debounce');
+const zigbeeHersdman = require('zigbee-herdsman');
+
+const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
+    'hvacThermostat', 'msTemperatureMeasurement'];
+
+// See zigbee-herdsman-converters
+const defaultBindGroup = {type: 'group_number', ID: 901};
+
+const defaultReportConfiguration = {
+    minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: 1,
+};
+
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') == null) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const reportClusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultReportConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultReportConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultReportConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultReportConfiguration},
+    ],
+};
+
+const pollOnMessage = [
+    {
+        // On messages that have the cluster and type of below
+        cluster: {
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 2}},
+                {type: 'commandHueNotification', data: {button: 3}},
+            ],
+            genLevelCtrl: [
+                {type: 'commandStep', data: {}},
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandStop', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMove', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
+        },
+        // Read the following attributes
+        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+    {
+        cluster: {
+            genLevelCtrl: [
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+                {type: 'commandToggle', data: {}},
+            ],
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+    {
+        cluster: {
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
+        },
+        read: {
+            cluster: 'lightingColorCtrl',
+            attributes: [],
+            // Since not all devices support the same attributes they need to be calculated dynamically
+            // depending on the capabilities of the endpoint.
+            attributesForEndpoint: async (endpoint) => {
+                const supportedAttrs = await getColorCapabilities(endpoint);
+                const readAttrs = [];
+                supportedAttrs.colorXY && readAttrs.push('currentX', 'currentY');
+                supportedAttrs.colorTemperature && readAttrs.push('colorTemperature');
+                return readAttrs;
+            },
+        },
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+];
+
+class Bind extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d), this.constructor.name);
+        this.pollDebouncers = {};
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let sourceKey = null;
+        let targetKey = null;
+        let clusters = null;
+        let skipDisableReporting = false;
+
+        if (this.legacyApi && topic.match(legacyTopicRegex)) {
+            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0];
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = message;
+        } else if (topic.match(topicRegex)) {
+            type = topic.endsWith('unbind') ? 'unbind' : 'bind';
+            message = JSON.parse(message);
+            sourceKey = message.from;
+            targetKey = message.to;
+            clusters = message.clusters;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
+        }
+
+        return {type, sourceKey, targetKey, clusters, skipDisableReporting};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {type, sourceKey, targetKey, clusters, skipDisableReporting} = this.parseMQTTMessage(topic, message);
+        if (!type) return null;
+        message = utils.parseJSON(message, message);
+
+        let error = null;
+        const source = this.zigbee.resolveEntityLegacy(sourceKey);
+        const target = targetKey === 'default_bind_group' ?
+            defaultBindGroup : this.zigbee.resolveEntityLegacy(targetKey);
+        const responseData = {from: sourceKey, to: targetKey};
+
+        if (!source || source.type !== 'device') {
+            error = `Source device '${sourceKey}' does not exist`;
+        } else if (!target) {
+            error = `Target device or group '${targetKey}' does not exist`;
+        } else {
+            const sourceName = source.settings.friendlyName;
+            const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+            const successfulClusters = [];
+            const failedClusters = [];
+            const attemptedClusters = [];
+
+            let bindTarget = null;
+            if (target.type === 'group') bindTarget = target.group;
+            else if (target.type === 'group_number') bindTarget = target.ID;
+            else bindTarget = target.endpoint;
+
+            // Find which clusters are supported by both the source and target.
+            // Groups are assumed to support all clusters.
+            for (const cluster of clusterCandidates) {
+                if (clusters && !clusters.includes(cluster)) continue;
+                let matchingClusters = false;
+
+                const anyClusterValid = target.type === 'group' || target.type === 'group_number' ||
+                    target.device.type === 'Coordinator';
+
+                if (!anyClusterValid) {
+                    matchingClusters = ((target.endpoint.supportsInputCluster(cluster) &&
+                            source.endpoint.supportsOutputCluster(cluster)) ||
+                            (source.endpoint.supportsInputCluster(cluster) &&
+                            target.endpoint.supportsOutputCluster(cluster)) );
+                }
+
+                const sourceValid = source.endpoint.supportsInputCluster(cluster) ||
+                source.endpoint.supportsOutputCluster(cluster);
+
+                if ( sourceValid && (anyClusterValid || matchingClusters)) {
+                    logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                    attemptedClusters.push(cluster);
+
+                    try {
+                        if (type === 'bind') {
+                            await source.endpoint.bind(cluster, bindTarget);
+                        } else {
+                            await source.endpoint.unbind(cluster, bindTarget);
+                        }
+
+                        successfulClusters.push(cluster);
+                        logger.info(
+                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                            `'${sourceName}' to '${targetName}'`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    } catch (error) {
+                        failedClusters.push(cluster);
+                        logger.error(
+                            `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                            `'${targetName}' (${error})`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}_failed`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    }
+                }
+            }
+
+            if (attemptedClusters.length === 0) {
+                logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+                error = `Nothing to ${type}`;
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                    );
+                }
+            } else if (failedClusters.length === attemptedClusters.length) {
+                error = `Failed to ${type}`;
+            }
+
+            responseData[`clusters`] = successfulClusters;
+            responseData[`failed`] = failedClusters;
+
+            if (successfulClusters.length !== 0) {
+                if (type === 'bind') {
+                    await this.setupReporting(source.endpoint.binds.filter((b) =>
+                        successfulClusters.includes(b.cluster.name) && b.target === bindTarget));
+                } else if ((target.type !== 'group_number') && !skipDisableReporting) {
+                    await this.disableUnnecessaryReportings(bindTarget);
+                }
+            }
+        }
+
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit(`devicesChanged`);
+        }
+    }
+
+    async groupMembersChanged(data) {
+        if (data.action === 'add') {
+            const bindsToGroup = this.zigbee.getClientsLegacy().map((c) => c.endpoints)
+                .reduce((a, v) => a.concat(v)).map((e) => e.binds)
+                .reduce((a, v) => a.concat(v)).filter((b) => b.target === data.group.group);
+            await this.setupReporting(bindsToGroup);
+        } else { // action === remove/remove_all
+            if (!data.skipDisableReporting) {
+                await this.disableUnnecessaryReportings(data.endpoint);
+            }
+        }
+    }
+
+    getSetupReportingEndpoints(bind, coordinatorEp) {
+        const endpoints = bind.target.constructor.name === 'Group' ? bind.target.members : [bind.target];
+        return endpoints.filter((e) => {
+            const supportsInputCluster = e.supportsInputCluster(bind.cluster.name);
+            const hasConfiguredReporting = !!e.configuredReportings.find((c) => c.cluster.name === bind.cluster.name);
+            const hasBind = !!e.binds.find((b) => b.cluster.name === bind.cluster.name && b.target === coordinatorEp);
+            return supportsInputCluster && !(hasBind && hasConfiguredReporting);
+        });
+    }
+
+    async setupReporting(binds) {
+        const coordinator = this.zigbee.getDevicesByTypeLegacy('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        for (const bind of binds.filter((b) => b.cluster.name in reportClusters)) {
+            for (const endpoint of this.getSetupReportingEndpoints(bind, coordinatorEndpoint)) {
+                const entity = `${this.zigbee.resolveEntityLegacy(endpoint.getDevice()).name}/${endpoint.ID}`;
+                try {
+                    await endpoint.bind(bind.cluster.name, coordinatorEndpoint);
+
+                    const items = await reportClusters[bind.cluster.name]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(bind.cluster.name, items);
+                    logger.info(`Succesfully setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                } catch (error) {
+                    logger.warn(`Failed to setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                }
+            }
+        }
+
+        this.eventBus.emit(`devicesChanged`);
+    }
+
+    async disableUnnecessaryReportings(target) {
+        const coordinator = this.zigbee.getDevicesByTypeLegacy('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        const endpoints = target.constructor.name === 'Group' ? target.members : [target];
+        for (const endpoint of endpoints) {
+            const entity = `${this.zigbee.resolveEntityLegacy(endpoint.getDevice()).name}/${endpoint.ID}`;
+            const boundClusters = endpoint.binds.filter((b) => b.target === coordinatorEndpoint)
+                .map((b) => b.cluster.name);
+            const requiredClusters = this.zigbee.getClientsLegacy().map((c) => c.endpoints)
+                .reduce((a, v) => a.concat(v))
+                .map((e) => e.binds).reduce((a, v) => a.concat(v)).filter((bind) => {
+                    if (bind.target.constructor.name === 'Group') {
+                        return bind.target.members.includes(endpoint);
+                    } else {
+                        return bind.target === endpoint;
+                    }
+                }).map((b) => b.cluster.name).filter((v, i, a) => a.indexOf(v) === i);
+
+            for (const cluster of boundClusters.filter((c) => !requiredClusters.includes(c) && c in reportClusters)) {
+                try {
+                    await endpoint.unbind(cluster, coordinatorEndpoint);
+                    const items = await reportClusters[cluster]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a, maximumReportInterval: 0xFFFF};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(cluster, items);
+                    logger.info(`Succesfully disabled reporting for '${entity}' cluster '${cluster}'`);
+                } catch (error) {
+                    logger.warn(`Failed to disable reporting for '${entity}' cluster '${cluster}'`);
+                }
+            }
+
+            this.eventBus.emit('reportingDisabled', {device: endpoint.getDevice()});
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'message') {
+            this.poll(data);
+        }
+    }
+
+    async poll(message) {
+        /**
+         * This method poll bound endpoints and group members for state changes.
+         *
+         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
+         * Hue bulbs only report their on/off state.
+         * When dimming the bulb via the dimmer switch the state is therefore not reported.
+         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
+         */
+        const polls = pollOnMessage.filter((p) =>
+            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
+            utils.equalsPartial(message.data, c.data)),
+        );
+
+        if (polls.length) {
+            let toPoll = [];
+
+            // Add bound devices
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
+            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
+            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
+            toPoll = toPoll.map((e) => e.target);
+
+            // If message is published to a group, add members of the group
+            const group = message.groupID !== 0 ? this.zigbee.getGroupByIDLegacy(message.groupID) : null;
+            if (group) {
+                toPoll = toPoll.concat(group.members);
+            }
+
+            toPoll = new Set(toPoll);
+
+            for (const endpoint of toPoll) {
+                for (const poll of polls) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID) ||
+                        !endpoint.supportsInputCluster(poll.read.cluster)) {
+                        continue;
+                    }
+
+                    let readAttrs = poll.read.attributes;
+                    if (poll.read.attributesForEndpoint) {
+                        const attrsForEndpoint = await poll.read.attributesForEndpoint(endpoint);
+                        readAttrs = [...poll.read.attributes, ...attrsForEndpoint];
+                    }
+
+                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
+                    if (!this.pollDebouncers[key]) {
+                        this.pollDebouncers[key] = debounce(async () => {
+                            await endpoint.read(poll.read.cluster, readAttrs);
+                        }, 1000);
+                    }
+
+                    this.pollDebouncers[key]();
+                }
+            }
+        }
+    }
+}
+
+module.exports = Bind;

lib/extension/bridge.ts

@@ -0,0 +1,671 @@
+/* eslint-disable camelcase */
+import logger from '../util/logger';
+import * as utils from '../util/utils';
+import * as settings from '../util/settings';
+import Transport from 'winston-transport';
+import bind from 'bind-decorator';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import objectAssignDeep from 'object-assign-deep';
+import {detailedDiff} from 'deep-object-diff';
+import ExtensionTS from './extensionts';
+import Device from '../model/device';
+import Group from '../model/group';
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class Bridge extends ExtensionTS {
+    private zigbee2mqttVersion: {commitHash: string, version: string};
+    private coordinatorVersion: CoordinatorVersion;
+    private restartRequired = false;
+    private lastJoinedDeviceIeeeAddr: string;
+    private requestLookup: {[key: string]: (message: KeyValue | string) => Promise<MQTTResponse>};
+
+    override async start(): Promise<void> {
+        this.requestLookup = {
+            'device/options': this.deviceOptions,
+            'device/configure_reporting': this.deviceConfigureReporting,
+            'device/remove': this.deviceRemove,
+            'device/rename': this.deviceRename,
+            'group/add': this.groupAdd,
+            'group/options': this.groupOptions,
+            'group/remove': this.groupRemove,
+            'group/rename': this.groupRename,
+            'permit_join': this.permitJoin,
+            'restart': this.restart,
+            'touchlink/factory_reset': this.touchlinkFactoryReset,
+            'touchlink/identify': this.touchlinkIdentify,
+            'touchlink/scan': this.touchlinkScan,
+            'health_check': this.healthCheck,
+            'options': this.bridgeOptions,
+            // Below are deprecated
+            'config/last_seen': this.configLastSeen,
+            'config/homeassistant': this.configHomeAssistant,
+            'config/elapsed': this.configElapsed,
+            'config/log_level': this.configLogLevel,
+        };
+
+        const mqtt = this.mqtt;
+        class EventTransport extends Transport {
+            log(info: {message: string, level: string}, callback: () => void): void {
+                const payload = stringify({message: info.message, level: info.level});
+                mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
+                callback();
+            }
+        }
+        // @ts-ignore TODO fix typing
+        logger.addTransport(new EventTransport());
+
+        this.zigbee2mqttVersion = await utils.getZigbee2MQTTVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.eventBus.onDeviceRenamed(this, () => this.publishInfo());
+        this.eventBus.onGroupMembersChanged(this, () => this.publishGroups());
+        this.eventBus.onDevicesChanged(this, () => this.publishDevices() && this.publishInfo());
+        this.eventBus.onPermitJoinChanged(this, () => !this.zigbee.isStopping() && this.publishInfo());
+
+        // Zigbee events
+        const publishEvent = (type: string, data: KeyValue): Promise<void> =>
+            this.mqtt.publish('bridge/event', stringify({type, data}), {retain: false, qos: 0});
+        this.eventBus.onDeviceJoined(this, (data) => {
+            this.lastJoinedDeviceIeeeAddr = data.device.ieeeAddr;
+            this.publishDevices();
+            publishEvent('device_joined', {friendly_name: data.device.name, ieee_address: data.device.ieeeAddr});
+        });
+        this.eventBus.onDeviceLeave(this, (data) => {
+            this.publishDevices();
+            publishEvent('device_leave',
+                {ieee_address: data.ieeeAddr, friendly_name: settings.getDevice(data.ieeeAddr)?.friendlyName});
+        });
+        this.eventBus.onDeviceNetworkAddressChanged(this, () => this.publishDevices());
+        this.eventBus.onDeviceInterview(this, (data) => {
+            this.publishDevices();
+            const payload: KeyValue =
+                {friendly_name: data.device.name, status: data.status, ieee_address: data.device.ieeeAddr};
+            if (data.status === 'successful') {
+                payload.supported = !!data.device.definition;
+                payload.definition = this.getDefinitionPayload(data.device);
+            }
+            publishEvent('device_interview', payload);
+        });
+        this.eventBus.onDeviceAnnounce(this, (data) => {
+            this.publishDevices();
+            publishEvent('device_announce', {friendly_name: data.device.name, ieee_address: data.device.ieeeAddr});
+        });
+
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+
+
+        this.onMQTTMessage_ = this.onMQTTMessage_.bind(this);
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+    }
+
+    // TODO remove trailing _
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        const match = data.topic.match(requestRegex);
+        const key = match?.[1]?.toLowerCase();
+        if (key in this.requestLookup) {
+            const message = utils.parseJSON(data.message, data.message);
+
+            try {
+                const response = await this.requestLookup[key](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${data.topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    @bind async deviceOptions(message: KeyValue | string): Promise<MQTTResponse> {
+        return this.changeEntityOptions('device', message);
+    }
+
+    @bind async groupOptions(message: KeyValue | string): Promise<MQTTResponse> {
+        return this.changeEntityOptions('group', message);
+    }
+
+    @bind async bridgeOptions(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || typeof message.options !== 'object') {
+            throw new Error(`Invalid payload`);
+        }
+
+        const diff: KeyValue = detailedDiff(settings.get(), message.options);
+
+        // Remove any settings that are in the deleted.diff but not in the passed options
+        const cleanupDeleted = (options: KeyValue, deleted: KeyValue): void => {
+            for (const key of Object.keys(deleted)) {
+                if (!(key in options)) {
+                    delete deleted[key];
+                } else if (!Array.isArray(options[key])) {
+                    cleanupDeleted(options[key], deleted[key]);
+                }
+            }
+        };
+        cleanupDeleted(message.options, diff.deleted);
+
+        const newSettings = objectAssignDeep({}, diff.added, diff.updated, diff.deleted);
+
+        // deep-object-diff converts arrays to objects, set original array back here
+        const convertBackArray = (before: KeyValue, after: KeyValue): void => {
+            for (const [key, afterValue] of Object.entries(after)) {
+                const beforeValue = before[key];
+                if (Array.isArray(beforeValue)) {
+                    after[key] = beforeValue;
+                } else if (afterValue && typeof beforeValue === 'object') {
+                    convertBackArray(beforeValue, afterValue);
+                }
+            }
+        };
+        convertBackArray(message.options, newSettings);
+
+        const restartRequired = settings.apply(newSettings);
+        if (restartRequired) this.restartRequired = true;
+
+        // Apply some settings on-the-fly.
+        if (newSettings.hasOwnProperty('permit_join')) {
+            await this.zigbee.permitJoin(newSettings.permit_join);
+        }
+
+        if (newSettings.hasOwnProperty('homeassistant')) {
+            await this.enableDisableExtension(newSettings.homeassistant, 'HomeAssistant');
+        }
+
+        if (newSettings.hasOwnProperty('advanced') && newSettings.advanced.hasOwnProperty('log_level')) {
+            // @ts-ignore fix type
+            logger.setLevel(newSettings.advanced.log_level);
+        }
+
+        logger.info('Succesfully changed options');
+        this.publishInfo();
+        return utils.getResponse(message, {restart_required: this.restartRequired}, null);
+    }
+
+    @bind async deviceRemove(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.removeEntity('device', message);
+    }
+
+    @bind async groupRemove(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.removeEntity('group', message);
+    }
+
+    @bind async healthCheck(message: string | KeyValue): Promise<MQTTResponse> {
+        return utils.getResponse(message, {healthy: true}, null);
+    }
+
+    @bind async groupAdd(message: string | KeyValue): Promise<MQTTResponse> {
+        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
+        const group = settings.addGroup(friendlyName, ID);
+        this.zigbee.createGroup(group.ID);
+        this.publishGroups();
+        return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
+    }
+
+    @bind async deviceRename(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.renameEntity('device', message);
+    }
+
+    @bind async groupRename(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.renameEntity('group', message);
+    }
+
+    @bind async restart(message: string | KeyValue): Promise<MQTTResponse> {
+        // Wait 500 ms before restarting so response can be send.
+        setTimeout(this.restartCallback, 500);
+        logger.info('Restarting Zigbee2MQTT');
+        return utils.getResponse(message, {}, null);
+    }
+
+    @bind async permitJoin(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        let value: boolean | string;
+        let time: number;
+        let device: Device = null;
+        if (typeof message === 'object') {
+            value = message.value;
+            time = message.time;
+            if (message.device) {
+                const resolved = this.zigbee.resolveEntity(message.device);
+                if (resolved instanceof Device) {
+                    device = resolved;
+                } else {
+                    throw new Error(`Device '${message.device}' does not exist`);
+                }
+            }
+        } else {
+            value = message;
+        }
+
+        if (typeof value === 'string') {
+            value = value.toLowerCase() === 'true';
+        }
+
+        await this.zigbee.permitJoin(value, device, time);
+        const response: {value: boolean, device?: string, time?: number} = {value};
+        if (device && typeof message === 'object') response.device = message.device;
+        if (time && typeof message === 'object') response.time = message.time;
+        return utils.getResponse(message, response, null);
+    }
+
+    // Deprecated
+    @bind async configLastSeen(message: KeyValue | string): Promise<MQTTResponse> {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        const value = this.getValue(message);
+        if (typeof value !== 'string' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'last_seen'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    @bind async configHomeAssistant(message: string | KeyValue): Promise<MQTTResponse> {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (typeof value !== 'boolean' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        await this.enableDisableExtension(value, 'HomeAssistant');
+        settings.set(['homeassistant'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    @bind async configElapsed(message: KeyValue | string): Promise<MQTTResponse> {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (typeof value !== 'boolean' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'elapsed'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    @bind async configLogLevel(message: KeyValue | string): Promise<MQTTResponse> {
+        const allowed = ['error', 'warn', 'info', 'debug'];
+        const value = this.getValue(message);
+        if (typeof value !== 'string' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        // @ts-ignore TODO fix type
+        logger.setLevel(value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    @bind async touchlinkIdentify(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
+            !message.hasOwnProperty('channel')) {
+            throw new Error('Invalid payload');
+        }
+
+        logger.info(`Start Touchlink identify of '${message.ieee_address}' on channel ${message.channel}`);
+        await this.zigbee.touchlinkIdentify(message.ieee_address, message.channel);
+        return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
+    }
+
+    @bind async touchlinkFactoryReset(message: KeyValue | string): Promise<MQTTResponse> {
+        let result = false;
+        const payload: {ieee_address?: string, channel?: number} = {};
+        if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
+            message.hasOwnProperty('channel')) {
+            logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
+            result = await this.zigbee.touchlinkFactoryReset(message.ieee_address, message.channel);
+            payload.ieee_address = message.ieee_address;
+            payload.channel = message.channel;
+        } else {
+            logger.info('Start Touchlink factory reset of first found device');
+            result = await this.zigbee.touchlinkFactoryResetFirst();
+        }
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            return utils.getResponse(message, payload, null);
+        } else {
+            logger.error('Failed to factory reset device through Touchlink');
+            throw new Error('Failed to factory reset device through Touchlink');
+        }
+    }
+
+    @bind async touchlinkScan(message: KeyValue | string): Promise<MQTTResponse> {
+        logger.info('Start Touchlink scan');
+        const result = await this.zigbee.touchlinkScan();
+        const found = result.map((r) => {
+            return {ieee_address: r.ieeeAddr, channel: r.channel};
+        });
+        logger.info('Finished Touchlink scan');
+        return utils.getResponse(message, {found}, null);
+    }
+
+    /**
+     * Utils
+     */
+
+    getValue(message: KeyValue | string): string | boolean | number {
+        if (typeof message === 'object') {
+            if (!message.hasOwnProperty('value')) {
+                throw new Error('No value given');
+            }
+
+            return message.value;
+        } else {
+            return message;
+        }
+    }
+
+    async changeEntityOptions(entityType: 'device' | 'group', message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const cleanup = (o: KeyValue): KeyValue => {
+            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
+            return o;
+        };
+
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        const oldOptions = objectAssignDeep({}, cleanup(entity.settings));
+        settings.changeEntityOptions(ID, message.options);
+        const newOptions = cleanup(settings.getEntity(ID));
+        await this.publishInfo();
+
+        logger.info(`Changed config for ${entityType} ${ID}`);
+
+        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+    }
+
+    @bind async deviceConfigureReporting(message: string | KeyValue): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('cluster') ||
+            !message.hasOwnProperty('maximum_report_interval') || !message.hasOwnProperty('minimum_report_interval') ||
+            !message.hasOwnProperty('reportable_change') || !message.hasOwnProperty('attribute')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const parsedID = utils.parseEntityID(message.id);
+        const endpoint = (this.getEntity('device', parsedID.ID) as Device).endpoint(parsedID.endpoint);
+
+        const coordinatorEndpoint = this.zigbee.getFirstCoordinatorEndpoint();
+        await endpoint.bind(message.cluster, coordinatorEndpoint);
+
+        await endpoint.configureReporting(message.cluster, [{
+            attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
+            maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
+        }]);
+
+        this.publishDevices();
+
+        logger.info(`Configured reporting for '${message.id}', '${message.cluster}.${message.attribute}'`);
+
+        return utils.getResponse(message, {
+            id: message.id, cluster: message.cluster, maximum_report_interval: message.maximum_report_interval,
+            minimum_report_interval: message.minimum_report_interval, reportable_change: message.reportable_change,
+            attribute: message.attribute,
+        }, null);
+    }
+
+    async renameEntity(entityType: 'group' | 'device', message: string | KeyValue): Promise<MQTTResponse> {
+        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
+        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
+            !message.hasOwnProperty('to')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
+            throw new Error('No device has joined since start');
+        }
+
+        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
+        const to = message.to;
+        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
+            message.homeassistant_rename : false;
+        const entity = this.getEntity(entityType, from);
+        const oldFriendlyName = entity.settings.friendlyName;
+
+        settings.changeFriendlyName(from, to);
+
+        // Clear retained messages
+        this.mqtt.publish(oldFriendlyName, '', {retain: true});
+
+        if (entity instanceof Device) {
+            this.publishDevices();
+            this.eventBus.emit(`deviceRenamed`, {device: entity, homeAssisantRename, from: oldFriendlyName, to});
+        } else {
+            this.publishGroups();
+            this.publishInfo();
+        }
+
+        // Repulish entity state
+        this.publishEntityState(to, {});
+
+        return utils.getResponse(
+            message,
+            {from: oldFriendlyName, to, homeassistant_rename: homeAssisantRename},
+            null,
+        );
+    }
+
+    async removeEntity(entityType: 'group' | 'device', message: string | KeyValue): Promise<MQTTResponse> {
+        const ID = typeof message === 'object' ? message.id : message.trim();
+        const entity = this.getEntity(entityType, ID);
+        const friendlyName = entity.name;
+        const entityID = entity.ID;
+
+        let block = false;
+        let force = false;
+        let blockForceLog = '';
+
+        if (entityType === 'device' && typeof message === 'object') {
+            block = !!message.block;
+            force = !!message.force;
+            blockForceLog = ` (block: ${block}, force: ${force})`;
+        } else if (entityType === 'group' && typeof message === 'object') {
+            force = !!message.force;
+            blockForceLog = ` (force: ${force})`;
+        }
+
+        try {
+            logger.info(`Removing ${entityType} '${entity.settings.friendlyName}'${blockForceLog}`);
+            if (entity instanceof Device) {
+                if (block) {
+                    settings.blockDevice(entity.ieeeAddr);
+                }
+
+                if (force) {
+                    await entity.zhDevice.removeFromDatabase();
+                } else {
+                    await entity.zhDevice.removeFromNetwork();
+                }
+            } else {
+                if (force) {
+                    entity.zhGroup.removeFromDatabase();
+                } else {
+                    await entity.zhGroup.removeFromNetwork();
+                }
+            }
+
+            // Fire event
+            if (entity instanceof Device) {
+                this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
+            }
+
+            // Remove from configuration.yaml
+            if (entity instanceof Device) {
+                settings.removeDevice(entityID as string);
+            } else {
+                settings.removeGroup(entityID);
+            }
+
+            // Remove from state
+            this.state.remove(entityID);
+
+            // Clear any retained messages
+            this.mqtt.publish(friendlyName, '', {retain: true});
+
+            logger.info(`Successfully removed ${entityType} '${friendlyName}'${blockForceLog}`);
+
+            if (entity instanceof Device) {
+                this.publishGroups();
+                this.publishDevices();
+                return utils.getResponse(message, {id: ID, block, force}, null);
+            } else {
+                this.publishGroups();
+                return utils.getResponse(message, {id: ID, force: force}, null);
+            }
+        } catch (error) {
+            throw new Error(
+                `Failed to remove ${entityType} '${friendlyName}'${blockForceLog} (${error})`,
+            );
+        }
+    }
+
+    getEntity(type: 'group' | 'device', ID: string): Device | Group {
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.constructor.name.toLowerCase() !== type) {
+            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
+        }
+        return entity;
+    }
+
+    async publishInfo(): Promise<void> {
+        const config = objectAssignDeep({}, settings.get());
+        delete config.advanced.network_key;
+        delete config.mqtt.password;
+        config.frontend && delete config.frontend.auth_token;
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
+            // @ts-ignore TODO fix type
+            log_level: logger.getLevel(),
+            permit_join: this.zigbee.getPermitJoin(),
+            permit_join_timeout: this.zigbee.getPermitJoinTimeout(),
+            restart_required: this.restartRequired,
+            config,
+            config_schema: settings.schema,
+        };
+
+        await this.mqtt.publish(
+            'bridge/info', stringify(payload), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    async publishDevices(): Promise<void> {
+        interface Data {
+            bindings: {cluster: string, target: {type: string, endpoint?: number, ieee_address?: string, id?: number}}[]
+            configured_reportings: {cluster: string, attribute: string | number,
+                minimum_report_interval: number, maximum_report_interval: number, reportable_change: number}[],
+            clusters: {input: string[], output: string[]}
+        }
+
+        const devices = this.zigbee.getDevices().map((device) => {
+            const endpoints: {[s: number]: Data} = {};
+            for (const endpoint of device.endpoints) {
+                const data: Data = {
+                    bindings: [],
+                    configured_reportings: [],
+                    clusters: {
+                        input: endpoint.getInputClusters().map((c) => c.name),
+                        output: endpoint.getOutputClusters().map((c) => c.name),
+                    },
+                };
+
+                for (const bind of endpoint.binds) {
+                    const target = utils.isEndpoint(bind.target) ?
+                        {type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID} :
+                        {type: 'group', id: bind.target.groupID};
+                    data.bindings.push({cluster: bind.cluster.name, target});
+                }
+
+                for (const configuredReporting of endpoint.configuredReportings) {
+                    data.configured_reportings.push({
+                        cluster: configuredReporting.cluster.name,
+                        attribute: configuredReporting.attribute.name || configuredReporting.attribute.ID,
+                        minimum_report_interval: configuredReporting.minimumReportInterval,
+                        maximum_report_interval: configuredReporting.maximumReportInterval,
+                        reportable_change: configuredReporting.reportableChange,
+                    });
+                }
+
+                endpoints[endpoint.ID] = data;
+            }
+
+            return {
+                ieee_address: device.ieeeAddr,
+                type: device.type,
+                network_address: device.networkAddress,
+                supported: !!device.definition,
+                friendly_name: device.name,
+                definition: this.getDefinitionPayload(device),
+                power_source: device.powerSource,
+                software_build_id: device.softwareBuildID,
+                date_code: device.dateCode,
+                model_id: device.modelID,
+                interviewing: device.interviewing,
+                interview_completed: device.interviewCompleted,
+                manufacturer: device.manufacturerName,
+                endpoints,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', stringify(devices),
+            {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    async publishGroups(): Promise<void> {
+        const groups = this.zigbee.getGroups().map((g) => {
+            return {
+                id: g.ID,
+                friendly_name: g.ID === 901 ? 'default_bind_group' : g.name,
+                members: g.members.map((e) => {
+                    return {ieee_address: e.getDevice().ieeeAddr, endpoint: e.ID};
+                }),
+            };
+        });
+        await this.mqtt.publish(
+            'bridge/groups', stringify(groups), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    getDefinitionPayload(device: Device):
+        {model: string, vendor: string, description: string, exposes: unknown[], supports_ota: boolean, icon: string} {
+        if (!device.definition) return null;
+        let icon = device.settings.icon ? device.settings.icon : device.definition.icon;
+        if (icon) {
+            icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(device.modelID));
+            icon = icon.replace('${model}', utils.sanitizeImageParameter(device.definition.model));
+        }
+        return {
+            model: device.definition.model,
+            vendor: device.definition.vendor,
+            description: device.definition.description,
+            exposes: device.definition.exposes,
+            supports_ota: !!device.definition.ota,
+            icon,
+        };
+    }
+}
+
+module.exports = Bridge;

lib/extension/bridgeConfig.js

@@ -1,289 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const utils = require('../util/utils');
-
-const configRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+))`);
-const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
-
-class BridgeConfig {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-
-        // Bind functions
-        this.permitJoin = this.permitJoin.bind(this);
-        this.lastSeen = this.lastSeen.bind(this);
-        this.elapsed = this.elapsed.bind(this);
-        this.reset = this.reset.bind(this);
-        this.logLevel = this.logLevel.bind(this);
-        this.devices = this.devices.bind(this);
-        this.groups = this.groups.bind(this);
-        this.rename = this.rename.bind(this);
-        this.remove = this.remove.bind(this);
-        this.ban = this.ban.bind(this);
-        this.deviceOptions = this.deviceOptions.bind(this);
-        this.addGroup = this.addGroup.bind(this);
-        this.removeGroup = this.removeGroup.bind(this);
-        this.whitelist= this.whitelist.bind(this);
-
-        // Set supported options
-        this.supportedOptions = {
-            'permit_join': this.permitJoin,
-            'last_seen': this.lastSeen,
-            'elapsed': this.elapsed,
-            'reset': this.reset,
-            'log_level': this.logLevel,
-            'devices': this.devices,
-            'groups': this.groups,
-            'devices/get': this.devices,
-            'rename': this.rename,
-            'remove': this.remove,
-            'ban': this.ban,
-            'device_options': this.deviceOptions,
-            'add_group': this.addGroup,
-            'remove_group': this.removeGroup,
-            'whitelist': this.whitelist,
-        };
-    }
-
-    whitelist(topic, message) {
-        message = message.toString();
-        const IDByFriendlyName = settings.getIeeeAddrByFriendlyName(message);
-        const deviceID = IDByFriendlyName ? IDByFriendlyName : message;
-        const whitelisted = settings.whitelistDevice(deviceID);
-        whitelisted ? logger.info(`Whitelisted '${deviceID}'`) : logger.error(`Failed to whitelist '${deviceID}'`);
-        this.mqtt.log('device_whitelisted', deviceID);
-    }
-
-    deviceOptions(topic, message) {
-        let json = null;
-        try {
-            json = JSON.parse(message.toString());
-        } catch (e) {
-            logger.error('Failed to parse message as JSON');
-            return;
-        }
-
-        if (!json.hasOwnProperty('friendly_name') || !json.hasOwnProperty('options')) {
-            logger.error('Invalid JSON message, should contain "friendly_name" and "options"');
-            return;
-        }
-
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(json.friendly_name);
-        if (!ieeeAddr) {
-            logger.error(`Failed to find device '${json.friendly_name}'`);
-            return;
-        }
-
-        settings.changeDeviceOptions(ieeeAddr, json.options);
-        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
-    }
-
-    permitJoin(topic, message) {
-        this.zigbee.permitJoin(message.toString().toLowerCase() === 'true', () => this.publish());
-    }
-
-    reset(topic, message) {
-        this.zigbee.softReset((error) => {
-            if (error) {
-                logger.error('Soft reset failed');
-            } else {
-                logger.info('Soft resetted ZNP');
-            }
-        });
-    }
-
-    lastSeen(topic, message) {
-        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
-        message = message.toString();
-
-        if (!allowed.includes(message)) {
-            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
-            return;
-        }
-
-        settings.set(['advanced', 'last_seen'], message);
-        logger.info(`Set last_seen to ${message}`);
-    }
-
-    elapsed(topic, message) {
-        const allowed = ['true', 'false'];
-        message = message.toString();
-
-        if (!allowed.includes(message)) {
-            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
-            return;
-        }
-
-        settings.set(['advanced', 'elapsed'], message === 'true');
-        logger.info(`Set elapsed to ${message}`);
-    }
-
-    logLevel(topic, message) {
-        const level = message.toString().toLowerCase();
-        if (allowedLogLevels.includes(level)) {
-            logger.info(`Switching log level to '${level}'`);
-            logger.transports.console.level = level;
-            logger.transports.file.level = level;
-        } else {
-            logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
-        }
-
-        this.publish();
-    }
-
-    devices(topic, message) {
-        const devices = this.zigbee.getDevices().map((device) => {
-            const payload = {
-                ieeeAddr: device.ieeeAddr,
-                type: device.type,
-            };
-
-            if (device.type !== 'Coordinator') {
-                const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-                const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                payload.model = mappedDevice ? mappedDevice.model : device.modelId;
-                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
-                payload.nwkAddr = device.nwkAddr;
-                payload.manufId = device.manufId;
-                payload.manufName = device.manufName;
-                payload.powerSource = device.powerSource;
-                payload.modelId = device.modelId;
-                payload.hwVersion = device.hwVersion;
-                payload.swBuildId = device.swBuildId;
-                payload.dateCode = device.dateCode;
-            }
-
-            return payload;
-        });
-
-        if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
-        } else {
-            this.mqtt.log('devices', devices);
-        }
-    }
-
-    groups(topic, message) {
-        this.mqtt.log('groups', settings.getGroups());
-    }
-
-    rename(topic, message) {
-        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} ` +
-                            `got ${message.toString()}`;
-
-        let json = null;
-        try {
-            json = JSON.parse(message.toString());
-        } catch (e) {
-            logger.error(invalid);
-            return;
-        }
-
-        // Validate message
-        if (!json.new || !json.old) {
-            logger.error(invalid);
-            return;
-        }
-
-        if (settings.changeFriendlyName(json.old, json.new)) {
-            logger.info(`Successfully renamed - ${json.old} to ${json.new} `);
-            this.mqtt.log('device_renamed', {from: json.old, to: json.new});
-        } else {
-            logger.error(`Failed to renamed - ${json.old} to ${json.new}`);
-            return;
-        }
-    }
-
-    addGroup(topic, message) {
-        const name = message.toString();
-        const added = settings.addGroup(name);
-        added ? logger.info(`Added group '${name}'`) : logger.error(`Failed to add group '${name}'`);
-    }
-
-    removeGroup(topic, message) {
-        const name = message.toString();
-        const removed = settings.removeGroup(name);
-        removed ? logger.info(`Removed group '${name}'`) : logger.error(`Failed to remove group '${name}'`);
-    }
-
-    remove(topic, message) {
-        this.removeOrBan(false, message);
-    }
-
-    ban(topic, message) {
-        this.removeOrBan(true, message);
-    }
-
-    removeOrBan(ban, message) {
-        message = message.toString();
-        const IDByFriendlyName = settings.getIeeeAddrByFriendlyName(message);
-        const deviceID = IDByFriendlyName ? IDByFriendlyName : message;
-        const device = this.zigbee.getDevice(deviceID);
-
-        const cleanup = () => {
-            // Remove from configuration.yaml
-            settings.removeDevice(deviceID);
-
-            // Remove from state
-            this.state.remove(deviceID);
-
-            logger.info(`Successfully ${ban ? 'banned' : 'removed'} ${deviceID}`);
-            this.mqtt.log(ban ? 'device_banned' : 'device_removed', message);
-        };
-
-        // Remove from zigbee network.
-        if (device) {
-            this.zigbee.removeDevice(deviceID, ban, (error) => {
-                if (!error) {
-                    cleanup();
-                } else {
-                    logger.error(`Failed to ${ban ? 'ban' : 'remove'} ${deviceID}`);
-                }
-            });
-        } else {
-            cleanup();
-        }
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.publish();
-    }
-
-    onMQTTMessage(topic, message) {
-        if (!topic.match(configRegex)) {
-            return false;
-        }
-
-        const option = topic.match(configRegex)[1];
-
-        if (!this.supportedOptions.hasOwnProperty(option)) {
-            return false;
-        }
-
-        this.supportedOptions[option](topic, message);
-
-        return true;
-    }
-
-    publish() {
-        utils.getZigbee2mqttVersion((info) => {
-            const topic = `bridge/config`;
-            const payload = {
-                version: info.version,
-                commit: info.commitHash,
-                coordinator: this.zigbee.getFirmwareVersion(),
-                log_level: logger.transports.console.level,
-                permit_join: this.zigbee.getPermitJoin(),
-            };
-
-            this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0}, null);
-        });
-    }
-}
-
-module.exports = BridgeConfig;

lib/extension/configure.ts

@@ -0,0 +1,140 @@
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import logger from '../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+import Device from '../model/device';
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends ExtensionTS {
+    private configuring = new Set();
+    private attempts: {[s: string]: number} = {};
+    private topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
+    private legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+
+    @bind private async onReportingDisabled(data: EventReportingDisabled): Promise<void> {
+        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
+        const device = this.zigbee.resolveEntity(data.device) as Device; // TODO assert device
+
+        if (device.zhDevice.meta?.hasOwnProperty('configured')) {
+            delete device.zhDevice.meta.configured;
+            device.zhDevice.save();
+        }
+
+        await this.configure(device, 'reporting_disabled');
+    }
+
+    // TODO remove trailing _
+    @bind private async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        if (data.topic === this.legacyTopic) {
+            const device = this.zigbee.resolveEntity(data.message);
+            if (!device || !(device instanceof Device)) {
+                logger.error(`Device '${data.message}' does not exist`);
+                return;
+            }
+
+            if (!device.definition || !device.definition.configure) {
+                logger.warn(`Skipping configure of '${device.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(device, 'mqtt_message', true);
+        } else if (data.topic === this.topic) {
+            const message = utils.parseJSON(data.message, data.message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const device = this.zigbee.resolveEntity(ID);
+            if (!device || !(device instanceof Device)) {
+                error = `Device '${ID}' does not exist`;
+            } else if (!device.definition || !device.definition.configure) {
+                error = `Device '${device.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(device, 'mqtt_message', true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
+        }
+    }
+
+    override async start(): Promise<void> {
+        for (const device of this.zigbee.getClients()) {
+            await this.configure(device, 'started');
+        }
+
+        this.eventBus.onDeviceJoined(this, (data) => {
+            if (data.device.zhDevice.meta.hasOwnProperty('configured')) {
+                delete data.device.zhDevice.meta.configured;
+                data.device.zhDevice.save();
+            }
+
+            this.configure(data.device, 'zigbee_event');
+        });
+        this.eventBus.onLastSeenChanged(this, (data) => this.configure(data.device, 'zigbee_event'));
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+        this.eventBus.onReportingDisabled(this, this.onReportingDisabled);
+    }
+
+    private async configure(device: Device, event: 'started' | 'zigbee_event' | 'reporting_disabled' | 'mqtt_message',
+        force=false, thowError=false): Promise<void> {
+        if (!force) {
+            if (!device.definition?.configure || device.zhDevice.interviewing) {
+                return;
+            }
+
+            if (device.zhDevice.meta?.hasOwnProperty('configured') &&
+                device.zhDevice.meta.configured === zhc.getConfigureKey(device.definition)) {
+                return;
+            }
+
+            // Only configure end devices when it is active, otherwise it will likely fails as they are sleeping.
+            if (device.zhDevice.type === 'EndDevice' && event !== 'zigbee_event') {
+                return;
+            }
+        }
+
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${device.name}'`);
+        try {
+            await device.definition.configure(device.zhDevice, this.zigbee.getFirstCoordinatorEndpoint(), logger);
+            logger.info(`Successfully configured '${device.name}'`);
+            device.zhDevice.meta.configured = zhc.getConfigureKey(device.definition);
+            device.zhDevice.save();
+            this.eventBus.emitDevicesChanged();
+            this.eventBus.emit(`devicesChanged`);
+        } catch (error) {
+            this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${device.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}
+
+module.exports = Configure;

lib/extension/deviceAvailability.js

@@ -1,182 +0,0 @@
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-
-// Some EndDevices should be pinged
-// e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const pingableDevices = [
-    zigbeeShepherdConverters.devices.find((d) => d.model === 'E11-G13'),
-];
-
-const toZigbeeCandidates = ['state', 'color', 'color_temp'];
-
-/**
- * This extensions pings devices to check if they are online.
- */
-class DeviceAvailability {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.availability_timeout = settings.get().advanced.availability_timeout;
-        this.timers = {};
-        this.pending = [];
-        this.state = {};
-
-        // Initialize blacklist
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => {
-            return settings.getIeeeAddrByFriendlyName(e) || e;
-        });
-    }
-
-    isPingable(device) {
-        logger.trace(`Checking if ${device.ieeeAddr} is pingable`);
-
-        if (this.blacklist.includes(device.ieeeAddr)) {
-            logger.trace(`${device.ieeeAddr} is not pingable because of blacklist`);
-            return false;
-        }
-
-        if (pingableDevices.find((d) => d.zigbeeModel.includes(device.modelId))) {
-            logger.trace(`${device.ieeeAddr} is pingable because in pingable devices`);
-            return true;
-        }
-
-        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
-        logger.trace(`${device.ieeeAddr} is pingable (${result}) not router or battery powered`);
-        return result;
-    }
-
-    getAllPingableDevices() {
-        return this.zigbee.getAllClients().filter((d) => this.isPingable(d));
-    }
-
-    onMQTTConnected() {
-        // As some devices are not checked for availability (e.g. battery powered devices)
-        // we mark all device as online by default.
-        this.zigbee.getDevices()
-            .filter((d) => d.type !== 'Coordinator')
-            .forEach((device) => this.publishAvailability(device.ieeeAddr, true));
-
-        // Start timers for all devices
-        this.getAllPingableDevices().forEach((device) => this.setTimer(device));
-    }
-
-    handleInterval(device) {
-        // Check if a job is already pending.
-        // This avoids overflowing of the queue in case the queue is not able to catch-up with the jobs being added.
-        const ieeeAddr = device.ieeeAddr;
-        if (this.pending.includes(ieeeAddr)) {
-            logger.debug(`Skipping ping for ${ieeeAddr} becuase job is already in queue`);
-            return;
-        }
-
-        this.pending.push(ieeeAddr);
-
-        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const errorLogLevel = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
-        const mechanism = utils.isXiaomiDevice(device) ? 'basic' : 'default';
-
-        this.zigbee.ping(ieeeAddr, errorLogLevel, (error) => {
-            this.publishAvailability(ieeeAddr, !error);
-
-            // Remove from pending jobs.
-            const index = this.pending.indexOf(ieeeAddr);
-            if (index !== -1) {
-                this.pending.splice(index, 1);
-            }
-
-            this.setTimer(device);
-        }, mechanism);
-    }
-
-    setTimer(device) {
-        if (this.timers[device.ieeeAddr]) {
-            clearTimeout(this.timers[device.ieeeAddr]);
-        }
-
-        this.timers[device.ieeeAddr] = setTimeout(() => {
-            this.handleInterval(device);
-        }, utils.secondsToMilliseconds(this.availability_timeout));
-    }
-
-    stop() {
-        this.zigbee.getDevices()
-            .filter((d) => d.type !== 'Coordinator')
-            .forEach((device) => this.publishAvailability(device.ieeeAddr, false));
-    }
-
-    onReconnect(ieeeAddr) {
-        const device = this.zigbee.getDevice(ieeeAddr);
-        let mappedDevice = null;
-
-        if (device && device.modelId) {
-            mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        }
-
-        if (mappedDevice) {
-            const converters = mappedDevice.toZigbee.filter((tz) => {
-                return tz.key.find((k) => toZigbeeCandidates.includes(k));
-            });
-
-            converters.forEach((converter) => {
-                const convertedResults = converter.convert(null, null, null, 'get');
-                if (convertedResults && convertedResults.length) {
-                    convertedResults.forEach((converted) => {
-                        this.zigbee.publish(
-                            ieeeAddr, 'device', converted.cid, converted.cmd, converted.cmdType,
-                            converted.zclData, converted.cfg, null, () => {}
-                        );
-                    });
-                }
-            });
-        }
-    }
-
-    publishAvailability(ieeeAddr, available) {
-        if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
-            this.onReconnect(ieeeAddr);
-        }
-
-        this.state[ieeeAddr] = available;
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : ieeeAddr;
-        const topic = `${name}/availability`;
-        const payload = available ? 'online' : 'offline';
-        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (device && this.isPingable(this.zigbee.getDevice(device.ieeeAddr))) {
-            // When a zigbee message from a device is received we know the device is still alive.
-            // => reset the timer.
-            this.setTimer(device);
-
-            const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
-            const offline = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr];
-
-            if (!online && !offline) {
-                // A new device has been connected
-                this.publishAvailability(device.ieeeAddr, true);
-            } else if (offline) {
-                // When a message is received and the device is marked as offline, mark it online.
-                this.publishAvailability(device.ieeeAddr, true);
-            } else if (online) {
-                /**
-                 * In case the device is powered off AND on within the availability timeout,
-                 * zigbee2qmtt does not detect the device as offline (device is still marked online).
-                 * When a device is turned on again the state could be out of sync.
-                 * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
-                 * endDeviceAnnce is typically send when a device comes online.
-                 *
-                 * This isn't needed for TRADFRI devices as they already send the state themself.
-                 */
-                if (message.type === 'endDeviceAnnce' && !utils.isIkeaTradfriDevice(device)) {
-                    this.onReconnect(device.ieeeAddr);
-                }
-            }
-        }
-    }
-}
-
-module.exports = DeviceAvailability;

lib/extension/deviceBind.js

@@ -1,128 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-
-const postfixes = utils.getPostfixes();
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
-
-const allowedClusters = [
-    5, // genScenes
-    6, // genOnOff
-    8, // genLevelCtrl
-    768, // lightingColorCtrl
-];
-
-class DeviceBind {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/+/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/+/+`);
-    }
-
-    getIDAndPostfix(topic) {
-        let postfix = null;
-        if (postfixes.find((p) => topic.endsWith(`/${p}`))) {
-            postfix = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-            // Remove postfix from topic
-            topic = topic.replace(`/${postfix}`, '');
-        }
-
-        return {ID: topic, postfix};
-    }
-
-    parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-
-        // Parse type from topic
-        const type = topic.split('/')[0];
-
-        // Remove type from topic
-        topic = topic.replace(`${type}/`, '');
-        return {type, ...this.getIDAndPostfix(topic)};
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Find source; can only be a device.
-        const sourceEntity = settings.resolveEntity(topic.ID);
-        const source = utils.getEndpointByEntityID(this.zigbee, sourceEntity.ID, topic.postfix);
-
-        const targetEntityIDPostfix= this.getIDAndPostfix(message.toString());
-        const targetEntity = settings.resolveEntity(targetEntityIDPostfix.ID);
-        let target = null;
-        if (targetEntity.type === 'device') {
-            target = utils.getEndpointByEntityID(this.zigbee, targetEntity.ID, targetEntityIDPostfix.postfix);
-        } else if (targetEntity.type === 'group') {
-            target = targetEntity.ID;
-        }
-
-        if (!source || !target) {
-            return false;
-        }
-
-        // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters (as we don't know which devices are in)
-        let supported = [];
-        if (targetEntity.type === 'device') {
-            supported = target.getSimpleDesc().inClusterList.filter((cluster) => {
-                return allowedClusters.includes(cluster);
-            });
-        } else if (targetEntity.type === 'group') {
-            supported = allowedClusters;
-        }
-
-        const clusters = source.getSimpleDesc().outClusterList.filter((cluster) => {
-            return supported.includes(cluster);
-        });
-
-        // Bind
-        clusters.forEach((cluster) => {
-            logger.debug(
-                `${topic.type}ing cluster '${cluster}' from '${sourceEntity.ID}' ` +
-                `(${source.epId}) to '${targetEntity.ID}'`
-            );
-
-            this.zigbee[topic.type](source, cluster, target, (error) => {
-                if (error) {
-                    logger.error(
-                        `Failed to ${topic.type} cluster '${cluster}' from '${sourceEntity.ID}' to ` +
-                        `'${targetEntity.ID}' (${error})`
-                    );
-                } else {
-                    logger.info(
-                        `Successfully ${topic.type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `'${sourceEntity.ID}' to '${targetEntity.ID}'`
-                    );
-
-                    this.mqtt.log(
-                        `device_${topic.type}`,
-                        {from: sourceEntity.ID, to: targetEntity.ID, cluster}
-                    );
-                }
-            });
-        });
-
-        return true;
-    }
-}
-
-module.exports = DeviceBind;

lib/extension/deviceConfigure.js

@@ -1,89 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const Queue = require('queue');
-
-/**
- * This extensions handles configuration of devices.
- */
-class DeviceConfigure {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.configured = {};
-        this.attempts = {};
-
-        this.queue = new Queue();
-        this.queue.concurrency = 1;
-        this.queue.autostart = true;
-    }
-
-    onZigbeeStarted() {
-        this.zigbee.getAllClients().forEach((device) => {
-            const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-
-            if (mappedDevice) {
-                this.configure(device.ieeeAddr, mappedDevice);
-            }
-        });
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (device && mappedDevice) {
-            const ieeeAddr = device.ieeeAddr;
-            if (!this.configured[ieeeAddr]) {
-                this.configure(ieeeAddr, mappedDevice);
-            }
-        }
-    }
-
-    configure(ieeeAddr, mappedDevice) {
-        if (!this.attempts.hasOwnProperty(ieeeAddr)) {
-            this.attempts[ieeeAddr] = 0;
-        }
-
-        if (mappedDevice.configure) {
-            const friendlyName = settings.getDevice(ieeeAddr) ? settings.getDevice(ieeeAddr).friendly_name : 'unknown';
-            // mark device as currently being configured to prevent race conditions
-            this.configured[ieeeAddr] = 'configuring';
-
-            this.queue.push((queueCallback) => {
-                logger.debug(`Configuring ${friendlyName} (${ieeeAddr}) ...`);
-
-                // Call configure function of this device.
-                mappedDevice.configure(ieeeAddr, this.zigbee.shepherd, this.zigbee.getCoordinator(), (ok, msg) => {
-                    if (ok) {
-                        logger.info(`Successfully configured ${friendlyName} (${ieeeAddr})`);
-                        this.configured[ieeeAddr] = 'configured';
-                    } else {
-                        // If a device is not joined in the network the configure command may fail
-                        // (especially, immediately after zigbee start)
-                        // also, this command may fail if the network is busy or lost data packets
-                        if (this.attempts[ieeeAddr] > 1) {
-                            // Give up after 3 attempts.
-                            this.configured[ieeeAddr] = 'failed';
-                        } else {
-                            // we're not done yet so mark not busy
-                            delete this.configured[ieeeAddr];
-                        }
-
-                        logger.warn(
-                            `Failed to configure ${friendlyName} (${ieeeAddr}) ('${msg}')` +
-                            ` (attempt #${this.attempts[ieeeAddr] + 1})`
-                        );
-
-                        logger.warn(`This can be ignored if the device is working properly`);
-
-                        this.attempts[ieeeAddr] += 1;
-                    }
-
-                    queueCallback();
-                });
-            });
-        } else {
-            // no need to configure anyting. So just mark as done
-            this.configured[ieeeAddr] = 'configured';
-        }
-    }
-}
-
-module.exports = DeviceConfigure;

lib/extension/deviceGroupMembership.js

@@ -1,73 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/.+/get_group_membership$`);
-
-class DeviceGroupMembership {
-    constructor(zigbee, mqtt, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.publishEntityState = publishEntityState;
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/device/+/get_group_membership`);
-    }
-
-    parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/device/`, '');
-
-        // Remove command from topic
-        topic = topic.replace(`/get_group_membership`, '');
-
-        return {friendlyName: topic};
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Map message to ieeeAddr and check if device exist.
-        const friendlyName = topic.friendlyName;
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(friendlyName) || friendlyName;
-        if (!this.zigbee.getDevice(ieeeAddr)) {
-            logger.error(`Failed to find device '${friendlyName}'`);
-            return;
-        }
-
-        const callback = (error, rsp) => {
-            if (error) {
-                logger.error(`Failed to get membership of ${ieeeAddr}`);
-            } else {
-                const {grouplist, capacity} = rsp;
-                const msgGroupList = `${ieeeAddr} is in groups [${grouplist}]`;
-                let msgCapacity;
-                if (capacity === 254) {
-                    msgCapacity = 'it can be a part of at least 1 more group';
-                } else {
-                    msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
-                }
-                logger.info(`${msgGroupList} and ${msgCapacity}`);
-
-                this.publishEntityState(ieeeAddr, {group_list: grouplist, group_capacity: capacity});
-            }
-        };
-
-        this.zigbee.publish(
-            ieeeAddr, 'device', 'genGroups', 'getMembership', 'functional',
-            {groupcount: 0, grouplist: []}, null, null, callback,
-        );
-
-        return true;
-    }
-}
-
-module.exports = DeviceGroupMembership;

lib/extension/devicePublish.js

@@ -1,235 +0,0 @@
-
-const settings = require('../util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const objectAssignDeep = require(`object-assign-deep`);
-
-const postfixes = utils.getPostfixes();
-const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
-
-const maxDepth = 20;
-
-const groupConverters = [
-    zigbeeShepherdConverters.toZigbeeConverters.light_onoff_brightness,
-    zigbeeShepherdConverters.toZigbeeConverters.light_colortemp,
-    zigbeeShepherdConverters.toZigbeeConverters.light_color,
-    zigbeeShepherdConverters.toZigbeeConverters.light_alert,
-    zigbeeShepherdConverters.toZigbeeConverters.ignore_transition,
-];
-
-class DevicePublish {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
-
-    onMQTTConnected() {
-        // Subscribe to topics.
-        const baseTopic = settings.get().mqtt.base_topic;
-        for (let step = 1; step < maxDepth; step++) {
-            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}set/+`);
-            this.mqtt.subscribe(`${topic}get`);
-            this.mqtt.subscribe(`${topic}get/+`);
-        }
-    }
-
-    parseTopic(topic) {
-        const match = topic.match(topicRegex);
-        if (!match) {
-            return null;
-        }
-
-        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
-        // If we didn't repalce base_topic we received something we don't care about
-        if (ID === match[1] || ID.match(/bridge\/config/)) {
-            return null;
-        }
-
-        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
-    }
-
-    handlePublishError(entity, message, error) {
-        const meta = {
-            entity,
-            message: message.toString(),
-        };
-
-        this.mqtt.log('zigbee_publish_error', error.toString(), meta);
-    }
-
-    handlePublished(entity, topic, converter, converted, key, value) {
-        if (topic.type === 'set' && converted.hasOwnProperty('newState') && converted.newState) {
-            const msg = converted.newState;
-
-            if (topic.postfix) {
-                msg[`state_${topic.postfix}`] = msg.state;
-                delete msg.state;
-            }
-
-            if (settings.get().advanced.last_seen !== 'disable') {
-                msg.last_seen = utils.formatDate(Date.now(), settings.get().advanced.last_seen);
-            }
-
-            this.publishEntityState(entity.ID, msg);
-        }
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Resolve the entity
-        const entity = settings.resolveEntity(topic.ID);
-
-        // Get entity details
-        let endpoint = null;
-        let converters = null;
-        let device = null;
-        let options = {};
-
-        if (entity.type === 'device') {
-            device = this.zigbee.getDevice(entity.ID);
-            if (!device) {
-                logger.error(`Failed to find device with ieeeAddr: '${entity.ID}'`);
-                this.mqtt.log('entity_not_found', entity.ID);
-                return;
-            }
-
-            // Map device to a model
-            const model = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-            if (!model) {
-                logger.warn(`Device with modelID '${device.modelId}' is not supported.`);
-                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-                return;
-            }
-
-            // Determine endpoint to publish to.
-            if (model.hasOwnProperty('ep')) {
-                const eps = model.ep(device);
-                endpoint = eps.hasOwnProperty(topic.postfix) ? eps[topic.postfix] : null;
-                if (endpoint === null && eps.hasOwnProperty('default')) {
-                    endpoint = eps['default'];
-                }
-            }
-
-            converters = model.toZigbee;
-
-            const deviceSettings = settings.getDevice(entity.ID);
-            options = objectAssignDeep.noMutate({}, model.options, deviceSettings);
-        } else if (entity.type === 'group') {
-            converters = groupConverters;
-            options = objectAssignDeep.noMutate({}, settings.getGroup(entity.ID));
-        }
-
-        // Convert the MQTT message to a Zigbee message.
-        let json = {};
-        if (topic.hasOwnProperty('attribute') && topic.attribute) {
-            json[topic.attribute] = message.toString();
-        } else {
-            try {
-                json = JSON.parse(message);
-            } catch (e) {
-                // Cannot be parsed to JSON, assume state message.
-                json = {state: message.toString()};
-            }
-        }
-
-        /**
-         * Home Assistant always publishes 'state', even when e.g. only setting
-         * the color temperature. This would lead to 2 zigbee publishes, where the first one
-         * (state) is probably unecessary.
-         */
-        if (settings.get().homeassistant) {
-            const deviceState = this.state.get(entity.ID);
-            const hasColorTemp = json.hasOwnProperty('color_temp');
-            const hasColor = json.hasOwnProperty('color');
-            const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
-            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
-                delete json.state;
-                logger.debug('Skipping state because of Home Assistant');
-            }
-        }
-
-        // Ensure that state and brightness are executed before other commands.
-        const keys = Object.keys(json);
-        keys.sort((a, b) => (['state', 'brightness'].includes(a) ? -1 : 1));
-
-        // For each key in the JSON message find the matching converter.
-        const usedConverters = [];
-        keys.forEach((key) => {
-            const converter = converters.find((c) => c.key.includes(key));
-
-            if (usedConverters.includes(converter)) {
-                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
-                return;
-            }
-
-            if (!converter) {
-                logger.error(`No converter available for '${key}' (${json[key]})`);
-                return;
-            }
-
-            // Converter didn't return a result, skip
-            const convertedResults = converter.convert(key, json[key], json, topic.type, topic.postfix, options);
-            if (!convertedResults || !convertedResults.length) {
-                return;
-            }
-
-            convertedResults.forEach((converted) => {
-                this.zigbee.publish(
-                    entity.ID,
-                    entity.type,
-                    converted.cid,
-                    converted.cmd,
-                    converted.cmdType,
-                    converted.zclData,
-                    converted.cfg,
-                    endpoint,
-                    (error, rsp) => {
-                        if (!error) {
-                            this.handlePublished(entity, topic, converter, converted, key, json[key]);
-                        } else {
-                            this.handlePublishError(entity, message, error);
-                        }
-                    }
-                );
-
-                // It's possible for devices to get out of sync when writing an attribute that's not reportable.
-                // So here we re-read the value after a specified timeout, this timeout could for example be the
-                // transition time of a color change or for forcing a state read for devices that don't
-                // automatically report a new state when set.
-                // When reporting is requested for a device (report: true in device-specific settings) we won't
-                // ever issue a read here, as we assume the device will properly report changes.
-                // Only do this when the retrieve_state option is enabled for this device.
-                const deviceSettings = settings.getDevice(entity.ID);
-                if (topic.type === 'set' && entity.type === 'device' &&
-                    converted.hasOwnProperty('readAfterWriteTime') && deviceSettings && deviceSettings.retrieve_state) {
-                    const getConvertedResults = converter.convert(key, json[key], json, 'get', '', options);
-                    getConvertedResults.forEach((getConverted) => {
-                        setTimeout(() => {
-                            this.zigbee.publish(
-                                entity.ID, entity.type, getConverted.cid, getConverted.cmd, getConverted.cmdType,
-                                getConverted.zclData, getConverted.cfg, endpoint, () => {}
-                            );
-                        }, converted.readAfterWriteTime);
-                    });
-                }
-
-                usedConverters.push(converter);
-            });
-        });
-
-        return true;
-    }
-}
-
-module.exports = DevicePublish;

lib/extension/deviceReceive.js

@@ -1,212 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const debounce = require('debounce');
-
-/**
- * This extensions handles messages received from devices.
- */
-class DeviceReceive {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-        this.coordinator = null;
-        this.elapsed = {};
-        this.debouncers = {};
-    }
-
-    onZigbeeStarted() {
-        this.coordinator = this.zigbee.getCoordinator().device.ieeeAddr;
-    }
-
-    publishDebounce(ieeeAddr, payload, time) {
-        if (!this.debouncers[ieeeAddr]) {
-            this.debouncers[ieeeAddr] = {
-                payload: {},
-                publish: debounce(() => {
-                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload);
-                    this.debouncers[ieeeAddr].payload = {};
-                }, time * 1000),
-            };
-        }
-
-        this.debouncers[ieeeAddr].payload = {...this.debouncers[ieeeAddr].payload, ...payload};
-        this.debouncers[ieeeAddr].publish();
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (message.type == 'devInterview') {
-            if (!settings.getDevice(message.data)) {
-                logger.info('Connecting with device...');
-                this.mqtt.log('pairing', 'connecting with device');
-            }
-
-            return;
-        }
-
-        if (message.type == 'devIncoming') {
-            logger.info('Device incoming...');
-            this.mqtt.log('pairing', 'device incoming');
-        }
-
-        if (!device) {
-            logger.warn('Message without device!');
-            return;
-        }
-
-        if (device.ieeeAddr === this.coordinator) {
-            logger.debug('Ignoring message from coordinator');
-            return;
-        }
-
-        // Check if this is a new device.
-        const settingsDevice = settings.getDevice(device.ieeeAddr);
-        if (!settingsDevice) {
-            logger.info(`New device '${device.modelId}' with address ${device.ieeeAddr} connected!`);
-            settings.addDevice(device.ieeeAddr);
-            this.mqtt.log('device_connected', device.ieeeAddr, {modelID: device.modelId});
-        }
-
-        if (!mappedDevice) {
-            logger.warn(`Device with modelID '${device.modelId}' is not supported.`);
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-            return;
-        }
-
-        // After this point we cant handle message withoud cid or cmdId anymore.
-        if (!message.data || (!message.data.cid && !message.data.cmdId)) {
-            return;
-        }
-
-        /**
-         * Don't handle re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The source address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the source address of the end device.
-         * As the same message is also received directly from the end device, it makes no sense
-         * to handle these messages.
-         */
-        const hasGroupID = message.hasOwnProperty('groupid') && message.groupid != 0;
-        if (utils.isXiaomiDevice(device) && utils.isRouter(device) && hasGroupID) {
-            logger.debug('Skipping re-transmitted Xiaomi message');
-            return;
-        }
-
-        // Find a conveter for this message.
-        const cid = message.data.cid;
-        const cmdId = message.data.cmdId;
-        const converters = mappedDevice.fromZigbee.filter((c) => {
-            if (cid) {
-                // readRsp messages have the same structure as attReport messages.
-                // search for attReport converters on readRsp.
-                if (c.cid === cid) {
-                    if (c.type instanceof Array) {
-                        return c.type.includes(message.type);
-                    } else {
-                        return c.type === message.type;
-                    }
-                }
-
-                return false;
-            } else if (cmdId) {
-                return c.cmd === cmdId;
-            }
-
-            return false;
-        });
-
-        // Check if there is an available converter
-        if (!converters.length) {
-            if (cid) {
-                // Don't log readRsp messages, they are not interesting most of the time.
-                if (message.type === 'readRsp') {
-                    return;
-                }
-
-                logger.warn(
-                    `No converter available for '${mappedDevice.model}' with cid '${cid}', ` +
-                    `type '${message.type}' and data '${JSON.stringify(message.data)}'`
-                );
-            } else if (cmdId) {
-                logger.warn(
-                    `No converter available for '${mappedDevice.model}' with cmd '${cmdId}' ` +
-                    `and data '${JSON.stringify(message.data)}'`
-                );
-            }
-
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
-            return;
-        }
-
-        // Convert this Zigbee message to a MQTT message.
-        // Get payload for the message.
-        // - If a payload is returned publish it to the MQTT broker
-        // - If NO payload is returned do nothing. This is for non-standard behaviour
-        //   for e.g. click switches where we need to count number of clicks and detect long presses.
-        const publish = (payload) => {
-            // Add device linkquality.
-            if (message.hasOwnProperty('linkquality')) {
-                payload.linkquality = message.linkquality;
-            }
-
-            // Add last seen timestamp
-            const now = Date.now();
-            if (settings.get().advanced.last_seen !== 'disable') {
-                payload.last_seen = utils.formatDate(now, settings.get().advanced.last_seen);
-            }
-
-            if (settings.get().advanced.elapsed) {
-                if (this.elapsed[device.ieeeAddr]) {
-                    payload.elapsed = now - this.elapsed[device.ieeeAddr];
-                }
-
-                this.elapsed[device.ieeeAddr] = now;
-            }
-
-            // Check if we have to debounce
-            if (settingsDevice && settingsDevice.hasOwnProperty('debounce')) {
-                this.publishDebounce(device.ieeeAddr, payload, settingsDevice.debounce);
-            } else {
-                this.publishEntityState(device.ieeeAddr, payload);
-
-                if (settings.get().homeassistant) {
-                    /**
-                     * Publish an empty value for click and action payload, in this way Home Assistant
-                     * can use Home Assistant entities in automations.
-                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
-                     */
-                    Object.keys(payload).forEach((key) => {
-                        if (['action', 'click'].includes(key)) {
-                            const counterPayload = {};
-                            counterPayload[key] = '';
-                            this.publishEntityState(device.ieeeAddr, counterPayload);
-                        }
-                    });
-                }
-            }
-        };
-
-        let payload = {};
-        converters.forEach((converter) => {
-            const options = {...settings.get().device_options, ...settings.getDevice(device.ieeeAddr)};
-            const converted = converter.convert(mappedDevice, message, publish, options);
-
-            if (converted) {
-                payload = {...payload, ...converted};
-            }
-        });
-
-        if (Object.keys(payload).length) {
-            publish(payload);
-        }
-    }
-}
-
-module.exports = DeviceReceive;

lib/extension/deviceReport.js

@@ -1,120 +0,0 @@
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
-const CC2530Router = zigbeeShepherdConverters.devices.find((d) => d.model === 'CC2530.ROUTER');
-const utils = require('../util/utils');
-
-const candidates = {
-    'genOnOff': {
-        attrs: ['onOff'],
-        reportIntervalMin: 3,
-        reportIntervalMax: 300,
-        reportableChange: 0,
-    },
-    'genLevelCtrl': {
-        attrs: ['currentLevel'],
-    },
-    'lightingColorCtrl': {
-        attrs: ['colorTemperature', 'currentX', 'currentY'],
-    },
-    'closuresWindowCovering': {
-        attrs: ['currentPositionLiftPercentage', 'currentPositionTiltPercentage'],
-        reportIntervalMin: 2,
-        reportIntervalMax: 300,
-    },
-};
-
-const reportInterval = {
-    min: 3,
-    max: 3600,
-};
-
-const reportableChange = 1;
-
-class DeviceReport {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
-
-    setupReporting(mappedDevice, device) {
-        let epId = null;
-
-        if (mappedDevice === CC2530Router) {
-            logger.debug('Not setting up reporting for CC2530 router');
-            return;
-        }
-
-        // Check if this device uses a different epId.
-        if (mappedDevice.hasOwnProperty('ep')) {
-            const eps = mappedDevice.ep(device);
-            epId = eps[''] || null;
-        }
-
-        const endpoint = this.zigbee.getEndpoint(device.ieeeAddr, epId);
-
-        if (!endpoint) {
-            logger.error(`Failed to setup reporting for ${device.ieeeAddr}, endpoint not found`);
-            return;
-        }
-
-        logger.debug(`Setting up reporting for ${device.ieeeAddr}`);
-        Object.values(endpoint.clusters).filter((c) => c).forEach((c) => {
-            const cluster = c.attrs.cid;
-            if (candidates[cluster]) {
-                const candidate = candidates[cluster];
-                let attributeNames = candidate.attrs.filter((a) => c.attrs.hasOwnProperty(a));
-
-                // Sometimes a cluster has no attributes, in this case setup reporting for all attributes.
-                attributeNames = attributeNames.length ? attributeNames : candidate.attrs;
-
-                const attributes = attributeNames.map((attribute) => {
-                    return {
-                        attr: attribute,
-                        min: candidate.hasOwnProperty('reportIntervalMin')?
-                            candidate.reportIntervalMin:reportInterval.min,
-                        max: candidate.hasOwnProperty('reportIntervalMax')?
-                            candidate.reportIntervalMax:reportInterval.max,
-                        change: candidate.hasOwnProperty('reportableChange')?
-                            candidate.reportableChange:reportableChange,
-                    };
-                });
-
-                if (attributes.length > 0) {
-                    this.zigbee.report(endpoint, cluster, attributes);
-                }
-            }
-        });
-    }
-
-    shouldSetupReporting(device) {
-        return utils.isRouter(device) && !utils.isBatteryPowered(device);
-    }
-
-    onZigbeeStarted() {
-        this.zigbee.getAllClients()
-            .filter((d) => this.shouldSetupReporting(d))
-            .forEach((device) => {
-                const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-
-                if (mappedDevice) {
-                    this.setupReporting(mappedDevice, device);
-                }
-            });
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        // Handle messages of type endDeviceAnnce and devIncoming.
-        // This message is typically send when a device comes online after being powered off
-        // Ikea TRADFRI tend to forget their reporting after powered off.
-        // Re-setup reporting.
-        // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (device && mappedDevice && ['endDeviceAnnce', 'devIncoming'].includes(message.type) &&
-            this.shouldSetupReporting(device)) {
-            this.setupReporting(mappedDevice, device);
-        }
-    }
-}
-
-module.exports = DeviceReport;

lib/extension/extension.js

@@ -1,8 +1,4 @@
-/**
- * This extensions is for documentation purposes only.
- * It describes all methods that are called by the controller.
- */
-class ExtensionTemplate {
+class Extension {
     /**
      * Besides intializing variables, the constructor should do nothing!
      *
@@ -10,31 +6,34 @@ class ExtensionTemplate {
      * @param {MQTT} mqtt MQTT controller
      * @param {State} state State controller
      * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
      */
-    constructor(zigbee, mqtt, state, publishEntityState) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.zigbee = zigbee;
         this.mqtt = mqtt;
         this.state = state;
         this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
     }
 
     /**
      * This method is called by the controller once Zigbee has been started.
      */
-    onZigbeeStarted() {}
+    // onZigbeeStarted() {}
 
     /**
      * This method is called by the controller once connected to the MQTT server.
      */
-    onMQTTConnected() {}
+    // onMQTTConnected() {}
 
     /**
      * Is called when a Zigbee message from a device is received.
-     * @param {Object?} message The received message (can be null)
-     * @param {Object?} device The device of the message (can be null)
-     * @param {Object?} mappedDevice The mapped device (can be null)
+     * @param {string} type Type of the message
+     * @param {Object} data Data of the message
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntityLegacy()
+     * @param {Object?} settingsDevice Device settings
      */
-    onZigbeeMessage(message, device, mappedDevice) {}
+    // onZigbeeEvent(type, data, resolvedEntity) {}
 
     /**
      * Is called when a MQTT message is received
@@ -42,14 +41,14 @@ class ExtensionTemplate {
      * @param {Object} message The received message
      * @return {boolean} if the message was handled
      */
-    onMQTTMessage(topic, message) {
-        return false;
-    }
+    // onMQTTMessage(topic, message) {}
 
     /**
      * Is called once the extension has to stop
      */
-    stop() {}
+    stop() {
+        this.eventBus.removeListeners(this.constructor.name);
+    }
 }
 
-module.exports = ExtensionTemplate;
+module.exports = Extension;

lib/extension/extensionts.ts

@@ -0,0 +1,51 @@
+// TODO: tempState -> State, rename to extension
+abstract class ExtensionTS {
+    protected zigbee: Zigbee;
+    protected mqtt: MQTT;
+    protected state: TempState;
+    protected publishEntityState: PublishEntityState;
+    protected eventBus: EventBus;
+    protected enableDisableExtension: (enable: boolean, name: string) => Promise<void>;
+    protected restartCallback: () => void;
+    protected addExtension: (extension: ExternalConverterClass) => void;
+
+    /**
+     * Besides intializing variables, the constructor should do nothing!
+     *
+     * @param {Zigbee} zigbee Zigbee controller
+     * @param {MQTT} mqtt MQTT controller
+     * @param {State} state State controller
+     * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
+     * @param {enableDisableExtension} enableDisableExtension Enable/disable extension method
+     * @param {restartCallback} restartCallback Restart Zigbee2MQTT
+     * @param {addExtension} addExtension Add an extension
+     */
+    constructor(zigbee: Zigbee, mqtt: MQTT, state: TempState, publishEntityState: PublishEntityState,
+        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => void, addExtension: (extension: ExternalConverterClass) => void) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+        this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
+        this.enableDisableExtension = enableDisableExtension;
+        this.restartCallback = restartCallback;
+        this.addExtension = addExtension;
+    }
+
+    /**
+     * Is called once the extension has to start
+     */
+    /* istanbul ignore next */
+    async start(): Promise<void> {}
+
+    /**
+     * Is called once the extension has to stop
+     */
+    async stop(): Promise<void> {
+        this.eventBus.removeListeners(this);
+    }
+}
+
+export default ExtensionTS;

lib/extension/externalConverters.ts

@@ -0,0 +1,17 @@
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import ExtensionTS from './extensionts';
+
+class ExternalConverters extends ExtensionTS {
+    override async start(): Promise<void> {
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zhc.addDeviceDefinition(toAdd);
+        }
+    }
+}
+
+module.exports = ExternalConverters;

lib/extension/externalExtension.ts

@@ -0,0 +1,111 @@
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import fs from 'fs';
+import * as data from './../util/data';
+import path from 'path';
+import logger from './../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import bind from 'bind-decorator';
+import ExtensionTS from './extensionts';
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
+
+class ExternalExtension extends ExtensionTS {
+    private requestLookup: {[s: string]: (message: KeyValue) => MQTTResponse};
+
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+        this.requestLookup = {'save': this.saveExtension, 'remove': this.removeExtension};
+        this.loadUserDefinedExtensions();
+        await this.publishExtensions();
+    }
+
+    private getExtensionsBasePath(): string {
+        return data.joinPath('extension');
+    }
+
+    private getListOfUserDefinedExtensions(): {name: string, code: string}[] {
+        const basePath = this.getExtensionsBasePath();
+        if (fs.existsSync(basePath)) {
+            return fs.readdirSync(basePath).filter((f) => f.endsWith('.js')).map((fileName) => {
+                const extensonFilePath = path.join(basePath, fileName);
+                return {'name': fileName, 'code': fs.readFileSync(extensonFilePath, 'utf-8')};
+            });
+        } else {
+            return [];
+        }
+    }
+
+    @bind private removeExtension(message: KeyValue): MQTTResponse {
+        const {name} = message;
+        const extensions = this.getListOfUserDefinedExtensions();
+        const extensionToBeRemoved = extensions.find((e) => e.name === name);
+
+        if (extensionToBeRemoved) {
+            this.enableDisableExtension(false, extensionToBeRemoved.name);
+            const basePath = this.getExtensionsBasePath();
+            const extensionFilePath = path.join(basePath, path.basename(name));
+            fs.unlinkSync(extensionFilePath);
+            this.publishExtensions();
+            logger.info(`Extension ${name} removed`);
+            return utils.getResponse(message, {}, null);
+        } else {
+            return utils.getResponse(message, {}, `Extension ${name} doesn't exists`);
+        }
+    }
+
+    @bind private saveExtension(message: KeyValue): MQTTResponse {
+        const {name, code} = message;
+        const ModuleConstructor = utils.loadModuleFromText(code) as ExternalConverterClass;
+        this.loadExtension(ModuleConstructor);
+        const basePath = this.getExtensionsBasePath();
+        /* istanbul ignore else */
+        if (!fs.existsSync(basePath)) {
+            fs.mkdirSync(basePath);
+        }
+        const extensonFilePath = path.join(basePath, path.basename(name));
+        fs.writeFileSync(extensonFilePath, code);
+        this.publishExtensions();
+        logger.info(`Extension ${name} loaded`);
+        return utils.getResponse(message, {}, null);
+    }
+
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        const match = data.topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            const message = utils.parseJSON(data.message, data.message) as KeyValue;
+            try {
+                const response = this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${data.topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    @bind private loadExtension(ConstructorClass: ExternalConverterClass): void {
+        this.enableDisableExtension(false, ConstructorClass.name);
+        this.addExtension(new ConstructorClass(
+            this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus, settings, logger));
+    }
+
+    private loadUserDefinedExtensions(): void {
+        const extensions = this.getListOfUserDefinedExtensions();
+        extensions
+            .map(({code}) => utils.loadModuleFromText(code))
+            .map(this.loadExtension);
+    }
+
+    private async publishExtensions(): Promise<void> {
+        const extensions = this.getListOfUserDefinedExtensions();
+        await this.mqtt.publish('bridge/extensions', stringify(extensions), {
+            retain: true,
+            qos: 0,
+        }, settings.get().mqtt.base_topic, true);
+    }
+}
+
+module.exports = ExternalExtension;

lib/extension/frontend.ts

@@ -0,0 +1,143 @@
+import http from 'http';
+import serveStatic from 'serve-static';
+import finalhandler from 'finalhandler';
+import logger from '../util/logger';
+// @ts-ignore
+import frontend from 'zigbee2mqtt-frontend';
+import WebSocket from 'ws';
+import net from 'net';
+import url from 'url';
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+
+/**
+ * This extension servers the frontend
+ */
+class Frontend extends ExtensionTS {
+    private mqttBaseTopic = settings.get().mqtt.base_topic;
+    private host = settings.get().frontend.host || '0.0.0.0';
+    private port = settings.get().frontend.port || 8080;
+    private authToken = settings.get().frontend.auth_token || false;
+    private retainedMessages = new Map();
+    private server: http.Server;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    private fileServer: serveStatic.RequestHandler<any>;
+    private wss: WebSocket.Server = null;
+
+    constructor(zigbee: Zigbee, mqtt: MQTT, state: TempState, publishEntityState: PublishEntityState,
+        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => void, addExtension: (extension: ExternalConverterClass) => void) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);
+        this.eventBus.onMQTTMessagePublished(this, this.onMQTTPublishMessage);
+    }
+
+    override async start(): Promise<void> {
+        this.server = http.createServer(this.onRequest);
+        this.server.on('upgrade', this.onUpgrade);
+
+        /* istanbul ignore next */ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const options = {setHeaders: (res: any, path: any): void => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+
+        this.fileServer = serveStatic(frontend.getPath(), options);
+        this.wss = new WebSocket.Server({noServer: true});
+        this.wss.on('connection', this.onWebSocketConnection);
+
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        for (const client of this.wss.clients) {
+            client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
+            client.terminate();
+        }
+        this.wss.close();
+        return new Promise((cb: () => void) => this.server.close(cb));
+    }
+
+    @bind private onRequest(request: http.IncomingMessage, response: http.ServerResponse): void {
+        // @ts-ignore
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+
+    private authenticate(request: http.IncomingMessage, cb: (authenticate: boolean) => void): void {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
+    }
+
+    @bind private onUpgrade(request: http.IncomingMessage, socket: net.Socket, head: Buffer): void {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
+    }
+
+    @bind private onWebSocketConnection(ws: WebSocket): void {
+        ws.on('message', (data: Buffer, isBinary: boolean) => {
+            if (!isBinary && data) {
+                const message = data.toString();
+                const {topic, payload} = JSON.parse(message);
+                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
+            }
+        });
+
+        for (const [key, value] of this.retainedMessages) {
+            ws.send(stringify({topic: key, payload: value}));
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            let payload: KeyValue = {};
+            if (this.state.exists(device.ieeeAddr)) {
+                payload = {...payload, ...this.state.get(device.ieeeAddr)};
+            }
+
+            const lastSeen = settings.get().advanced.last_seen;
+            /* istanbul ignore if */
+            if (lastSeen !== 'disable') {
+                payload.last_seen = utils.formatDate(device.lastSeen, lastSeen);
+            }
+
+            if (device.zhDevice.linkquality !== undefined) {
+                payload.linkquality = device.zhDevice.linkquality;
+            }
+
+            ws.send(stringify({topic: device.name, payload}));
+        }
+    }
+
+    @bind private onMQTTPublishMessage(data: EventMQTTMessagePublished): void {
+        if (data.topic.startsWith(`${this.mqttBaseTopic}/`)) {
+            // Send topic without base_topic
+            const topic = data.topic.substring(this.mqttBaseTopic.length + 1);
+            const payload = utils.parseJSON(data.payload, data.payload);
+            if (data.options.retain) {
+                this.retainedMessages.set(topic, payload);
+            }
+
+            if (this.wss) {
+                for (const client of this.wss.clients) {
+                    /* istanbul ignore else */
+                    if (client.readyState === WebSocket.OPEN) {
+                        client.send(stringify({topic, payload}));
+                    }
+                }
+            }
+        }
+    }
+}
+
+module.exports = Frontend;

lib/extension/groups.js

@@ -1,241 +1,377 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const data = require('../util/data');
+const Extension = require('./extension');
 const utils = require('../util/utils');
-const fs = require('fs');
-const diff = require('deep-diff');
+const stringify = require('json-stable-stringify-without-jsonify');
+const equals = require('fast-deep-equal/es6');
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/.+/(remove|add|remove_all)$`);
-const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(remove|add|remove_all)$`);
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
-class Groups {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
+const stateProperties = {
+    'state': (value, definition) => true,
+    'brightness': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' &&
+            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
+    'color_mode': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && (
+            (e.features.find((f) => f.name === `color_${value}`)) ||
+            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
+};
+
+class Groups extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
-
-        this.groupsCacheFile = data.joinPathStorage('groups_cache.json');
-        this.groupsCache = this.readGroupsCache();
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.lastOptimisticState = {};
     }
 
-    readGroupsCache() {
-        return fs.existsSync(this.groupsCacheFile) ? JSON.parse(fs.readFileSync(this.groupsCacheFile, 'utf8')) : {};
+    async onZigbeeStarted() {
+        this.eventBus.on('stateChange', this.onStateChange, this.constructor.name);
+        await this.syncGroupsWithSettings();
     }
 
-    writeGroupsCache() {
-        fs.writeFileSync(this.groupsCacheFile, JSON.stringify(this.groupsCache), 'utf8');
-    }
+    async syncGroupsWithSettings() {
+        const settingsGroups = settings.getGroups();
+        const zigbeeGroups = this.zigbee.getGroupsLegacy();
 
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/add`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove_all`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
-    }
-
-    apply(from, to) {
-        const sortGroups = (obj) => Object.keys(obj).forEach((key) => obj[key] = obj[key].sort());
-
-        sortGroups(from);
-        sortGroups(to);
-
-        const differences = diff(from, to);
-        if (differences) {
-            differences.forEach((diff) => {
-                const groupID = diff.path[0];
-
-                if (diff.kind === 'N') {
-                    diff.rhs.forEach((ID) => this.updateDeviceGroup(ID, 'add', groupID));
-                } else if (diff.kind === 'A') {
-                    if (diff.item.lhs) {
-                        this.updateDeviceGroup(diff.item.lhs, 'remove', groupID);
-                    } else {
-                        this.updateDeviceGroup(diff.item.rhs, 'add', groupID);
-                    }
-                } else if (diff.kind === 'D') {
-                    diff.lhs.forEach((ID) => this.updateDeviceGroup(ID, 'remove', groupID));
-                } else if (diff.kind === 'E') {
-                    this.updateDeviceGroup(diff.rhs, 'add', groupID);
-                    this.updateDeviceGroup(diff.lhs, 'remove', groupID);
-                }
-            });
-        }
-    }
-
-    parseID(ID) {
-        let entityID = ID;
-        let endpointID = null;
-        const postfix = utils.getPostfixes().find((p) => entityID.endsWith(`/${p}`));
-        if (postfix) {
-            // Found a postfix, retrieve the endpoint which correspodns to the postfix
-            entityID = entityID.substring(0, entityID.length - (postfix.length + 1));
-            const endpoint = utils.getEndpointByEntityID(this.zigbee, entityID, postfix);
-
-            if (!endpoint) {
-                return;
-            }
-
-            endpointID = endpoint.epId;
-        }
-
-        return {endpointID, entityID};
-    }
-
-    getGroupsOfDevice(entityID) {
-        return Object.keys(settings.getGroups()).filter((groupID) => {
-            return settings.getGroup(groupID).devices.includes(entityID);
-        });
-    }
-
-    onStateChange(ieeeAddr, from, to) {
-        const properties = ['state', 'brightness', 'color_temp', 'color'];
-        const payload = {};
-
-        properties.forEach((prop) => {
-            if (to.hasOwnProperty(prop) && (!from || from[prop] != to[prop])) {
-                payload[prop] = to[prop];
-            }
-        });
-
-        if (Object.keys(payload)) {
-            const groups = this.getGroupsOfDevice(ieeeAddr);
-            groups.forEach((groupID) => {
-                this.publishEntityState(groupID, payload);
-            });
-        }
-    }
-
-    onZigbeeStarted() {
-        this.state.registerOnStateChangeListener(this.onStateChange);
-
-        const settingsGroups = {};
-        Object.keys(settings.getGroups()).forEach((groupID) => {
-            settingsGroups[groupID] = settings.getGroup(groupID).devices;
-        });
-
-        this.apply(this.groupsCache, settingsGroups);
-    }
-
-    parseTopic(topic) {
-        if (!topic.match(topicRegex) && !topic.match(topicRegexRemoveAll)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/group/`, '');
-
-        // Parse type from topic
-        const type = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-        // Remove type from topic
-        topic = topic.replace(`/${type}`, '');
-
-        return {friendly_name: type === 'remove_all' ? null : topic, type};
-    }
-
-    updateDeviceGroup(ID, cmd, groupID) {
-        let payload = null;
-        if (cmd === 'add') {
-            payload = {groupid: groupID, groupname: ''};
-            cmd = 'add';
-        } else if (cmd === 'remove') {
-            payload = {groupid: groupID};
-            cmd = 'remove';
-        } else if (cmd === 'remove_all') {
-            payload = {};
-            cmd = 'removeAll';
-        }
-
-        const {entityID, endpointID} = this.parseID(ID);
-        const ieeeAddr = settings.resolveEntity(entityID).ID;
-
-        const cb = (error, rsp) => {
-            if (error) {
-                logger.error(`Failed to ${cmd} ${ieeeAddr} from ${groupID}`);
-            } else {
-                logger.info(`Successfully ${cmd} ${ieeeAddr} to ${groupID}`);
-
-                // Log to MQTT
-                const lookup = {
-                    'add': 'device_group_add',
-                    'remove': 'device_group_remove',
-                    'removeAll': 'device_group_remove_all',
-                };
-
-                this.mqtt.log({
-                    type: lookup[cmd],
-                    message: {
-                        device: settings.getDevice(ieeeAddr).friendly_name,
-                        group: groupID,
-                    },
-                });
-
-                // Update group cache
-                if (cmd === 'add') {
-                    if (!this.groupsCache[groupID]) {
-                        this.groupsCache[groupID] = [];
-                    }
-
-                    if (!this.groupsCache[groupID].includes(ieeeAddr)) {
-                        this.groupsCache[groupID].push(ieeeAddr);
-                    }
-                } else if (cmd === 'remove') {
-                    if (this.groupsCache[groupID]) {
-                        this.groupsCache[groupID] = this.groupsCache[groupID].filter((device) => device != ieeeAddr);
-                    }
-                } else if (cmd === 'removeAll') {
-                    Object.keys(this.groupsCache).forEach((groupID_) => {
-                        this.groupsCache[groupID_] = this.groupsCache[groupID_].filter((device) => device != ieeeAddr);
-                    });
-                }
-
-                this.writeGroupsCache();
-
-                // Update settings
-                if (cmd === 'add') {
-                    settings.addDeviceToGroup(groupID, ieeeAddr);
-                } else if (cmd === 'remove') {
-                    settings.removeDeviceFromGroup(groupID, ieeeAddr);
-                } else if (cmd === 'removeAll') {
-                    Object.keys(settings.get().groups).forEach((groupID_) => {
-                        settings.removeDeviceFromGroup(groupID_, ieeeAddr);
-                    });
+        const addRemoveFromGroup = async (action, deviceName, groupName, endpoint, group) => {
+            try {
+                logger.info(`${action === 'add' ? 'Adding' : 'Removing'} '${deviceName}' to group '${groupName}'`);
+                if (action === 'remove') {
+                    await endpoint.removeFromGroup(group);
+                } else {
+                    await endpoint.addToGroup(group);
                 }
+            } catch (error) {
+                logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+                logger.debug(error.stack);
             }
         };
 
-        this.zigbee.publish(
-            ieeeAddr, 'device', 'genGroups', cmd, 'functional',
-            payload, null, endpointID, cb,
-        );
-    }
+        for (const settingGroup of settingsGroups) {
+            const groupID = settingGroup.ID;
+            const zigbeeGroup = zigbeeGroups.find((g) =>
+                g.groupID === groupID) || this.zigbee.createGroupLegacy(groupID);
+            const settingsEntity = settingGroup.devices.map((d) => {
+                const entity = this.zigbee.resolveEntityLegacy(d);
+                if (!entity) logger.error(`Cannot find '${d}' of group '${settingGroup.friendlyName}'`);
+                return entity;
+            }).filter((e) => e != null);
 
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Find ID of this group.
-        let groupID = null;
-        if (topic.type !== 'remove_all') {
-            groupID = settings.getGroupIDByFriendlyName(topic.friendly_name);
-            if (!groupID) {
-                logger.error(`Group with friendly_name '${topic.friendly_name}' doesn't exist`);
-                return;
+            // In settings but not in zigbee
+            for (const entity of settingsEntity) {
+                if (!zigbeeGroup.hasMember(entity.endpoint)) {
+                    addRemoveFromGroup('add', entity.name, settingGroup.friendlyName, entity.endpoint, zigbeeGroup);
+                }
             }
 
-            groupID = groupID.toString();
+            // In zigbee but not in settings
+            for (const endpoint of zigbeeGroup.members) {
+                if (!settingsEntity.find((e) => e.endpoint === endpoint)) {
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, settingGroup.friendlyName, endpoint, zigbeeGroup);
+                }
+            }
         }
 
-        // Send command to the device.
-        this.updateDeviceGroup(message.toString(), topic.type, groupID);
+        for (const zigbeeGroup of zigbeeGroups) {
+            if (!settingsGroups.find((g) => g.ID === zigbeeGroup.groupID)) {
+                for (const endpoint of zigbeeGroup.members) {
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, zigbeeGroup.groupID, endpoint, zigbeeGroup);
+                }
+            }
+        }
+    }
 
+    async onStateChange(data) {
+        const reason = 'group_optimistic';
+        if (data.reason === reason) {
+            return;
+        }
+
+        const payload = {};
+
+        let endpointName;
+        for (let [prop, value] of Object.entries(data.update)) {
+            const endpointNameMatch = utils.endpointNames.find((n) => prop.endsWith(`_${n}`));
+            if (endpointNameMatch) {
+                prop = prop.substring(0, prop.length - endpointNameMatch.length - 1);
+                endpointName = endpointNameMatch;
+            }
+
+            if (prop in stateProperties) {
+                payload[prop] = value;
+            }
+        }
+
+        if (Object.keys(payload).length) {
+            const resolvedEntity =
+                this.zigbee.resolveEntityLegacy(`${data.ID}${endpointName ? `/${endpointName}` : ''}`);
+            const zigbeeGroups = this.zigbee.getGroupsLegacy().filter((zigbeeGroup) => {
+                const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
+                return settingsGroup && (!settingsGroup.hasOwnProperty('optimistic') || settingsGroup.optimistic);
+            });
+
+            if (resolvedEntity.type === 'device') {
+                for (const zigbeeGroup of zigbeeGroups) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint) &&
+                        !equals(this.lastOptimisticState[zigbeeGroup.groupID], payload)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                            this.lastOptimisticState[zigbeeGroup.groupID] = payload;
+                            await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
+                        }
+                    }
+                }
+            } else {
+                // Invalidate the last optimistic group state when group state is changed directly.
+                delete this.lastOptimisticState[resolvedEntity.group.groupID];
+
+                const groupIDsToPublish = new Set();
+                for (const member of resolvedEntity.group.members) {
+                    const resolvedEntity = this.zigbee.resolveEntityLegacy(member);
+                    const memberPayload = {};
+                    Object.keys(payload).forEach((key) => {
+                        if (stateProperties[key](payload[key], resolvedEntity.definition)) {
+                            memberPayload[key] = payload[key];
+                        }
+                    });
+
+                    const endpointName = resolvedEntity.endpointName;
+                    if (endpointName) {
+                        Object.keys(memberPayload).forEach((key) => {
+                            memberPayload[`${key}_${endpointName}`] = memberPayload[key];
+                            delete memberPayload[key];
+                        });
+                    }
+
+                    await this.publishEntityState(member.getDevice().ieeeAddr, memberPayload, reason);
+                    for (const zigbeeGroup of zigbeeGroups) {
+                        if (zigbeeGroup.hasMember(member)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                                groupIDsToPublish.add(zigbeeGroup.groupID);
+                            }
+                        }
+                    }
+                }
+                groupIDsToPublish.delete(resolvedEntity.group.groupID);
+                for (const groupID of groupIDsToPublish) {
+                    await this.publishEntityState(groupID, payload, reason);
+                }
+            }
+        }
+    }
+
+    areAllMembersOff(zigbeeGroup) {
+        for (const member of zigbeeGroup.members) {
+            const device = member.getDevice();
+            if (this.state.exists(device.ieeeAddr)) {
+                const state = this.state.get(device.ieeeAddr);
+                if (state && state.state === 'ON') {
+                    return false;
+                }
+            }
+        }
         return true;
     }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let resolvedEntityGroup = null;
+        let resolvedEntityDevice = null;
+        let hasEndpointName = null;
+        let error = null;
+        let groupKey = null;
+        let deviceKey = null;
+        let triggeredViaLegacyApi = false;
+        let skipDisableReporting = false;
+
+        /* istanbul ignore else */
+        const topicRegexMatch = topic.match(topicRegex);
+        const legacyTopicRegexRemoveAllMatch = topic.match(legacyTopicRegexRemoveAll);
+        const legacyTopicRegexMatch = topic.match(legacyTopicRegex);
+
+        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
+            triggeredViaLegacyApi = true;
+            if (legacyTopicRegexMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntityLegacy(legacyTopicRegexMatch[1]);
+                type = legacyTopicRegexMatch[2];
+
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {
+                            friendly_name: message, group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists',
+                        };
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return {};
+                }
+            } else {
+                type = 'remove_all';
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntityLegacy(message);
+            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
+                logger.error(`Device '${message}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {
+                        friendly_name: message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
+                    };
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
+                }
+
+                return {};
+            }
+
+            hasEndpointName = utils.endpointNames.find((p) => message.endsWith(`/${p}`));
+        } else if (topicRegexMatch) {
+            type = topicRegexMatch[1];
+            message = JSON.parse(message);
+            deviceKey = message.device;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
+
+            if (type !== 'remove_all') {
+                groupKey = message.group;
+                resolvedEntityGroup = this.zigbee.resolveEntityLegacy(message.group);
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    error = `Group '${message.group}' does not exist`;
+                }
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntityLegacy(message.device);
+            if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
+                error = `Device '${message.device}' does not exist`;
+            }
+
+            hasEndpointName = utils.endpointNames.find((p) => message.device.endsWith(`/${p}`));
+        }
+
+        return {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
+            triggeredViaLegacyApi, skipDisableReporting,
+        };
+    }
+
+    async onMQTTMessage(topic, message) {
+        let {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
+            groupKey, deviceKey, skipDisableReporting,
+        } = this.parseMQTTMessage(topic, message);
+        if (!type) return;
+        message = utils.parseJSON(message, message);
+
+        const responseData = {device: deviceKey};
+        if (groupKey) {
+            responseData.group = groupKey;
+        }
+
+        if (!error) {
+            try {
+                const keys = [
+                    `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+                    `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
+                ];
+
+                const definition = resolvedEntityDevice.definition;
+                const endpoints = definition && definition.endpoint ?
+                    definition.endpoint(resolvedEntityDevice.device) : null;
+                let endpointName = null;
+                if (endpoints) {
+                    const matches = Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID);
+                    endpointName = matches && matches[0];
+                }
+
+                if (endpointName) {
+                    keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+                    keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+                }
+
+                if (!hasEndpointName) {
+                    keys.push(resolvedEntityDevice.name);
+                    keys.push(resolvedEntityDevice.device.ieeeAddr);
+                }
+
+                if (type === 'add') {
+                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+                    settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_add`, message: payload}),
+                        );
+                    }
+                } else if (type === 'remove') {
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+                    settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_remove`, message: payload}),
+                        );
+                    }
+                } else { // remove_all
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    await resolvedEntityDevice.endpoint.removeFromAllGroups();
+                    for (const settingsGroup of settings.getGroups()) {
+                        settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const payload = {friendly_name: resolvedEntityDevice.name};
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_group_remove_all`, message: payload}),
+                            );
+                        }
+                    }
+                }
+            } catch (e) {
+                error = `Failed to ${type} from group (${e.message})`;
+                logger.debug(e.stack);
+            }
+        }
+
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit('groupMembersChanged',
+                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint,
+                    skipDisableReporting: skipDisableReporting});
+        }
+    }
 }
 
 module.exports = Groups;

lib/extension/legacy/availability.js

@@ -0,0 +1,227 @@
+const logger = require('../../util/logger');
+const settings = require('../../util/settings');
+const utils = require('../../util/utils');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const Extension = require('../extension');
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/(.*)/availability`);
+
+// Pingable end devices, some end devices should be pinged
+// e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
+const pingableEndDevices = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-N1EA'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
+];
+
+const Hours25 = 1000 * 60 * 60 * 25;
+const AvailabilityLagRatio = 0.1;
+
+
+function timeoutLag(timeout, ratio) {
+    const lag = timeout * ratio;
+    return Math.floor(Math.random() * Math.floor(lag));
+}
+
+/**
+ * This extensions pings devices to check if they are online.
+ */
+class AvailabilityLegacy extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.availability_timeout = settings.get().advanced.availability_timeout;
+        this.timers = {};
+        this.state = {};
+
+        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
+        this.eventBus.on('deviceRenamed', (data) => this.onDeviceRenamed(data), this.constructor.name);
+
+        this.blocklist = settings.get().advanced.availability_blocklist
+            .concat(settings.get().advanced.availability_blacklist)
+            .map((e) => settings.getEntity(e).ID);
+
+        this.passlist = settings.get().advanced.availability_passlist
+            .concat(settings.get().advanced.availability_whitelist)
+            .map((e) => settings.getEntity(e).ID);
+    }
+
+    onDeviceRenamed(data) {
+        this.mqtt.publish(`${data.from}/availability`, null, {retain: true, qos: 0});
+    }
+
+    onDeviceRemoved(resolvedEntity) {
+        this.mqtt.publish(`${resolvedEntity.name}/availability`, null, {retain: true, qos: 0});
+        delete this.state[resolvedEntity.device.ieeeAddr];
+        clearTimeout(this.timers[resolvedEntity.device.ieeeAddr]);
+    }
+
+    inPasslistOrNotInBlocklist(device) {
+        const ieeeAddr = device.ieeeAddr;
+        const deviceSettings = settings.getDevice(ieeeAddr);
+        const name = deviceSettings && deviceSettings.friendlyName;
+
+        // Passlist is not empty and device is in it, enable availability
+        if (this.passlist.length > 0) {
+            return this.passlist.includes(ieeeAddr) || (name && this.passlist.includes(name));
+        }
+
+        // Device is on blocklist, disable availability
+        if (this.blocklist.includes(ieeeAddr) || (name && this.blocklist.includes(name))) {
+            return false;
+        }
+
+        return true;
+    }
+
+    isPingable(device) {
+        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
+            return true;
+        }
+
+        // Device is a mains powered router
+        return device.type === 'Router' && device.powerSource !== 'Battery';
+    }
+
+    onMQTTConnected() {
+        for (const device of this.zigbee.getClientsLegacy()) {
+            // Mark all devices as online on start
+            const ieeeAddr = device.ieeeAddr;
+            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
+
+            if (this.inPasslistOrNotInBlocklist(device)) {
+                if (this.isPingable(device)) {
+                    this.setTimerPingable(device);
+                } else {
+                    this.timers[ieeeAddr] = setInterval(() => {
+                        this.handleIntervalNotPingable(device);
+                    }, utils.seconds(300));
+                }
+            }
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        // Clear topics for non-existing devices
+        const match = topic.match(topicRegex);
+        if (match && (!this.zigbee.resolveEntityLegacy(match[1]) ||
+            this.zigbee.resolveEntityLegacy(match[1]).name !== match[1])) {
+            this.mqtt.publish(`${match[1]}/availability`, null, {retain: true, qos: 0});
+        }
+    }
+
+    async handleIntervalPingable(device) {
+        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device.ieeeAddr);
+        if (!resolvedEntity) {
+            logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
+            return;
+        }
+
+        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
+        try {
+            await device.ping();
+            this.publishAvailability(device, true);
+            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
+        } catch (error) {
+            this.publishAvailability(device, false);
+            logger[level](`Failed to ping '${resolvedEntity.name}'`);
+        } finally {
+            this.setTimerPingable(device);
+        }
+    }
+
+    async handleIntervalNotPingable(device) {
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device.ieeeAddr);
+        if (!resolvedEntity || !device.lastSeen) {
+            return;
+        }
+
+        const ago = Date.now() - resolvedEntity.device.lastSeen;
+        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
+
+        if (ago > Hours25) {
+            this.publishAvailability(device, false);
+        }
+    }
+
+    setTimerPingable(device) {
+        const timeout = this.availability_timeout + timeoutLag(this.availability_timeout, AvailabilityLagRatio);
+        clearTimeout(this.timers[device.ieeeAddr]);
+        this.timers[device.ieeeAddr] = setTimeout(async () => {
+            await this.handleIntervalPingable(device);
+        }, utils.seconds(timeout));
+    }
+
+    async stop() {
+        super.stop();
+        for (const timer of Object.values(this.timers)) {
+            clearTimeout(timer);
+        }
+
+        this.zigbee.getClientsLegacy().forEach((device) => this.publishAvailability(device, false));
+    }
+
+    async onReconnect(device) {
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device);
+        if (resolvedEntity && resolvedEntity.definition) {
+            try {
+                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
+                    if (converter) {
+                        await converter.convertGet(device.endpoints[0], key, {});
+                    }
+                }
+            } catch (error) {
+                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
+            }
+        }
+    }
+
+    publishAvailability(device, available, force=false) {
+        const ieeeAddr = device.ieeeAddr;
+        if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
+            this.onReconnect(device);
+        }
+
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device);
+        const topic = `${resolvedEntity.name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        if (this.state[ieeeAddr] !== available || force) {
+            this.state[ieeeAddr] = available;
+            this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+        if (!device) {
+            return;
+        }
+
+        if (this.inPasslistOrNotInBlocklist(device)) {
+            this.publishAvailability(data.device, true);
+
+            if (this.isPingable(device)) {
+                // When a zigbee message from a device is received we know the device is still alive.
+                // => reset the timer.
+                this.setTimerPingable(device);
+
+                const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
+                if (online && type === 'deviceAnnounce' && !utils.isIkeaTradfriDevice(device)) {
+                    /**
+                     * In case the device is powered off AND on within the availability timeout,
+                     * zigbee2qmtt does not detect the device as offline (device is still marked online).
+                     * When a device is turned on again the state could be out of sync.
+                     * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
+                     * deviceAnnounce is typically send when a device comes online.
+                     *
+                     * This isn't needed for TRADFRI devices as they already send the state themself.
+                     */
+                    this.onReconnect(device);
+                }
+            }
+        }
+    }
+}
+
+module.exports = AvailabilityLegacy;

lib/extension/legacy/bridgeLegacy.js

@@ -0,0 +1,473 @@
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const utils = require('../../util/utils');
+const assert = require('assert');
+const Extension = require('../extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+const configRegex =
+    new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
+const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
+
+class BridgeLegacy extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        // Bind functions
+        this.permitJoin = this.permitJoin.bind(this);
+        this.lastSeen = this.lastSeen.bind(this);
+        this.elapsed = this.elapsed.bind(this);
+        this.reset = this.reset.bind(this);
+        this.logLevel = this.logLevel.bind(this);
+        this.devices = this.devices.bind(this);
+        this.groups = this.groups.bind(this);
+        this.rename = this.rename.bind(this);
+        this.renameLast = this.renameLast.bind(this);
+        this.remove = this.remove.bind(this);
+        this.forceRemove = this.forceRemove.bind(this);
+        this.ban = this.ban.bind(this);
+        this.deviceOptions = this.deviceOptions.bind(this);
+        this.addGroup = this.addGroup.bind(this);
+        this.removeGroup = this.removeGroup.bind(this);
+        this.whitelist = this.whitelist.bind(this);
+        this.touchlinkFactoryReset = this.touchlinkFactoryReset.bind(this);
+
+        this.lastJoinedDeviceName = null;
+
+        // Set supported options
+        this.supportedOptions = {
+            'permit_join': this.permitJoin,
+            'last_seen': this.lastSeen,
+            'elapsed': this.elapsed,
+            'reset': this.reset,
+            'log_level': this.logLevel,
+            'devices': this.devices,
+            'groups': this.groups,
+            'devices/get': this.devices,
+            'rename': this.rename,
+            'rename_last': this.renameLast,
+            'remove': this.remove,
+            'force_remove': this.forceRemove,
+            'ban': this.ban,
+            'device_options': this.deviceOptions,
+            'add_group': this.addGroup,
+            'remove_group': this.removeGroup,
+            'force_remove_group': this.removeGroup,
+            'whitelist': this.whitelist,
+            'touchlink/factory_reset': this.touchlinkFactoryReset,
+        };
+    }
+
+    whitelist(topic, message) {
+        try {
+            const entity = settings.getEntity(message);
+            assert(entity, `Entity '${message}' does not exist`);
+            settings.whitelistDevice(entity.ID);
+            logger.info(`Whitelisted '${entity.friendlyName}'`);
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to whitelist '${message}' '${error}'`);
+        }
+    }
+
+    deviceOptions(topic, message) {
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error('Failed to parse message as JSON');
+            return;
+        }
+
+        if (!json.hasOwnProperty('friendly_name') || !json.hasOwnProperty('options')) {
+            logger.error('Invalid JSON message, should contain "friendly_name" and "options"');
+            return;
+        }
+
+        const entity = settings.getEntity(json.friendly_name);
+        assert(entity, `Entity '${json.friendly_name}' does not exist`);
+        settings.changeEntityOptions(entity.ID, json.options);
+        logger.info(`Changed device specific options of '${json.friendly_name}' (${stringify(json.options)})`);
+    }
+
+    async permitJoin(topic, message) {
+        await this.zigbee.permitJoinLegacy(message.toLowerCase() === 'true');
+        this.publish();
+    }
+
+    async reset(topic, message) {
+        try {
+            await this.zigbee.reset('soft');
+            logger.info('Soft resetted ZNP');
+        } catch (error) {
+            logger.error('Soft reset failed');
+        }
+    }
+
+    lastSeen(topic, message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'last_seen'], message);
+        logger.info(`Set last_seen to ${message}`);
+    }
+
+    elapsed(topic, message) {
+        const allowed = ['true', 'false'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'elapsed'], message === 'true');
+        logger.info(`Set elapsed to ${message}`);
+    }
+
+    logLevel(topic, message) {
+        const level = message.toLowerCase();
+        if (allowedLogLevels.includes(level)) {
+            logger.info(`Switching log level to '${level}'`);
+            logger.setLevel(level);
+        } else {
+            logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
+        }
+
+        this.publish();
+    }
+
+    async devices(topic, message) {
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const devices = this.zigbee.getDevicesLegacy().map((device) => {
+            const payload = {
+                ieeeAddr: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+            };
+
+            if (device.type !== 'Coordinator') {
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const friendlyDevice = settings.getDevice(device.ieeeAddr);
+                payload.model = definition ? definition.model : device.modelID;
+                payload.vendor = definition ? definition.vendor : '-';
+                payload.description = definition ? definition.description : '-';
+                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
+                payload.manufacturerID = device.manufacturerID;
+                payload.manufacturerName = device.manufacturerName;
+                payload.powerSource = device.powerSource;
+                payload.modelID = device.modelID;
+                payload.hardwareVersion = device.hardwareVersion;
+                payload.softwareBuildID = device.softwareBuildID;
+                payload.dateCode = device.dateCode;
+                payload.lastSeen = device.lastSeen;
+            } else {
+                payload.friendly_name = 'Coordinator';
+                payload.softwareBuildID = coordinator.type;
+                payload.dateCode = coordinator.meta.revision.toString();
+                payload.lastSeen = Date.now();
+            }
+
+            return payload;
+        });
+
+        if (topic.split('/').pop() == 'get') {
+            this.mqtt.publish(
+                `bridge/config/devices`, stringify(devices), {}, settings.get().mqtt.base_topic, false, false,
+            );
+        } else {
+            this.mqtt.publish('bridge/log', stringify({type: 'devices', message: devices}));
+        }
+    }
+
+    groups(topic, message) {
+        const payload = settings.getGroups().map((g) => {
+            const group = {...g};
+            delete group.friendlyName;
+            return group;
+        });
+
+        this.mqtt.publish('bridge/log', stringify({type: 'groups', message: payload}));
+    }
+
+    rename(topic, message) {
+        const invalid =
+            `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
+
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error(invalid);
+            return;
+        }
+
+        // Validate message
+        if (!json.new || !json.old) {
+            logger.error(invalid);
+            return;
+        }
+
+        this._renameInternal(json.old, json.new);
+    }
+
+    renameLast(topic, message) {
+        if (!this.lastJoinedDeviceName) {
+            logger.error(`Cannot rename last joined device, no device has joined during this session`);
+            return;
+        }
+
+        this._renameInternal(this.lastJoinedDeviceName, message);
+    }
+
+    _renameInternal(from, to) {
+        try {
+            const isGroup = settings.getGroup(from) !== null;
+            settings.changeFriendlyName(from, to);
+            logger.info(`Successfully renamed - ${from} to ${to} `);
+            const entity = this.zigbee.resolveEntityLegacy(to);
+            const eventData = isGroup ? {group: entity.group} : {device: entity.device, from, to};
+            eventData.homeAssisantRename = false;
+            this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
+
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to rename - ${from} to ${to}`);
+        }
+    }
+
+    addGroup(topic, message) {
+        let id = null;
+        let name = null;
+        try {
+            // json payload with id and friendly_name
+            const json = JSON.parse(message);
+            if (json.hasOwnProperty('id')) {
+                id = json.id;
+                name = `group_${id}`;
+            }
+            if (json.hasOwnProperty('friendly_name')) {
+                name = json.friendly_name;
+            }
+        } catch (e) {
+            // just friendly_name
+            name = message;
+        }
+
+        if (name == null) {
+            logger.error('Failed to add group, missing friendly_name!');
+            return;
+        }
+
+        const group = settings.addGroup(name, id);
+        this.zigbee.createGroupLegacy(group.ID);
+        this.mqtt.publish('bridge/log', stringify({type: `group_added`, message: name}));
+        logger.info(`Added group '${name}'`);
+    }
+
+    removeGroup(topic, message) {
+        const name = message;
+        const entity = this.zigbee.resolveEntityLegacy(message);
+        assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+
+        if (topic.includes('force')) {
+            entity.group.removeFromDatabase();
+        } else {
+            entity.group.removeFromNetwork();
+        }
+        settings.removeGroup(message);
+
+        this.mqtt.publish('bridge/log', stringify({type: `group_removed`, message}));
+        logger.info(`Removed group '${name}'`);
+    }
+
+    async forceRemove(topic, message) {
+        await this.removeForceRemoveOrBan('force_remove', message);
+    }
+
+    async remove(topic, message) {
+        await this.removeForceRemoveOrBan('remove', message);
+    }
+
+    async ban(topic, message) {
+        await this.removeForceRemoveOrBan('ban', message);
+    }
+
+    async removeForceRemoveOrBan(action, message) {
+        const entity = this.zigbee.resolveEntityLegacy(message.trim());
+        const lookup = {
+            ban: ['banned', 'Banning', 'ban'],
+            force_remove: ['force_removed', 'Force removing', 'force remove'],
+            remove: ['removed', 'Removing', 'remove'],
+        };
+
+        if (!entity) {
+            logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
+
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            return;
+        }
+
+        const cleanup = () => {
+            // Fire event
+            this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
+
+            // Remove from configuration.yaml
+            settings.removeDevice(entity.settings.ID);
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}`, message}));
+        };
+
+        try {
+            logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+            if (action === 'force_remove') {
+                await entity.device.removeFromDatabase();
+            } else {
+                await entity.device.removeFromNetwork();
+            }
+
+            cleanup();
+        } catch (error) {
+            logger.error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+            // eslint-disable-next-line
+            logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
+
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
+        }
+
+        if (action === 'ban') {
+            settings.banDevice(entity.settings.ID);
+        }
+    }
+
+    async onMQTTConnected() {
+        await this.publish();
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (!topic.match(configRegex)) {
+            return false;
+        }
+
+        const option = topic.match(configRegex)[1];
+
+        if (!this.supportedOptions.hasOwnProperty(option)) {
+            return false;
+        }
+
+        await this.supportedOptions[option](topic, message);
+
+        return true;
+    }
+
+    async publish() {
+        const info = await utils.getZigbee2MQTTVersion();
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const topic = `bridge/config`;
+        const payload = {
+            version: info.version,
+            commit: info.commitHash,
+            coordinator,
+            network: await this.zigbee.getNetworkParameters(),
+            log_level: logger.getLevel(),
+            permit_join: this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0});
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceName = resolvedEntity.name;
+        }
+
+        if (type === 'deviceJoined') {
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+            );
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                    );
+                } else {
+                    const meta = {friendly_name: resolvedEntity.name, supported: false};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `pairing`, message: 'interview_successful', meta}),
+                    );
+                }
+            } else if (data.status === 'failed') {
+                const meta = {friendly_name: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    stringify({type: `pairing`, message: 'interview_failed', meta}),
+                );
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `pairing`, message: 'interview_started', meta}),
+                    );
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            const meta = {friendly_name: resolvedEntity.name};
+            this.mqtt.publish('bridge/log', stringify({type: `device_announced`, message: 'announce', meta}));
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const meta = {friendly_name: name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    stringify({type: `device_removed`, message: 'left_network', meta}),
+                );
+            }
+        }
+    }
+
+    async touchlinkFactoryReset() {
+        logger.info('Starting touchlink factory reset...');
+        this.mqtt.publish(
+            'bridge/log',
+            stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+        );
+        const result = await this.zigbee.touchlinkFactoryResetFirst();
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+            );
+        } else {
+            logger.warn('Failed to factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+            );
+        }
+    }
+}
+
+module.exports = BridgeLegacy;

lib/extension/legacy/deviceGroupMembership.js

@@ -0,0 +1,50 @@
+/* istanbul ignore file */
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const Extension = require('../extension');
+
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
+
+class DeviceGroupMembership extends Extension {
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const entity = this.zigbee.resolveEntityLegacy(match[1]);
+        if (!entity || entity.type !== 'device') {
+            logger.error(`Device '${match[1]}' does not exist`);
+            return;
+        }
+
+        const response = await entity.endpoint.command(
+            `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
+        );
+
+        if (!response) {
+            logger.warn(`Couldn't get group membership of ${entity.device.ieeeAddr}`);
+            return;
+        }
+
+        let {grouplist, capacity} = response;
+
+        grouplist = grouplist.map((gid) => {
+            const g = settings.getGroup(gid);
+            return g ? g.friendlyName : gid;
+        });
+
+        const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
+        let msgCapacity;
+        if (capacity === 254) {
+            msgCapacity = 'it can be a part of at least 1 more group';
+        } else {
+            msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
+        }
+        logger.info(`${msgGroupList} and ${msgCapacity}`);
+
+        this.publishEntityState(entity.device.ieeeAddr, {group_list: grouplist, group_capacity: capacity});
+    }
+}
+
+module.exports = DeviceGroupMembership;

lib/extension/legacy/report.js

@@ -0,0 +1,193 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const logger = require('../../util/logger');
+const settings = require('../../util/settings');
+const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
+
+const defaultConfiguration = {
+    minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
+};
+
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0115'),
+];
+
+const reportKey = 1;
+
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const clusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultConfiguration},
+    ],
+};
+
+class Report extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.queue = new Set();
+        this.failed = new Set();
+        this.enabled = settings.get().advanced.report;
+    }
+
+    shouldIgnoreClusterForDevice(cluster, definition) {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+            // Device announces it but doesn't support it
+            // https://github.com/Koenkk/zigbee2mqtt/issues/2611
+            return true;
+        }
+
+        return false;
+    }
+
+    async setupReporting(resolvedEntity) {
+        const {device, definition} = resolvedEntity;
+
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
+
+        const term1 = this.enabled ? 'Setup' : 'Disable';
+        const term2 = this.enabled ? 'setup' : 'disabled';
+
+        try {
+            for (const ep of device.endpoints) {
+                for (const [cluster, configuration] of Object.entries(clusters)) {
+                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
+                        logger.debug(`${term1} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                const toAdd = {...entry};
+                                if (!this.enabled) toAdd.maximumReportInterval = 0xFFFF;
+                                items.push(toAdd);
+                                delete items[items.length - 1].condition;
+                            }
+                        }
+
+                        this.enabled ?
+                            await ep.bind(cluster, this.coordinatorEndpoint) :
+                            await ep.unbind(cluster, this.coordinatorEndpoint);
+
+                        await ep.configureReporting(cluster, items);
+                        logger.info(
+                            `Successfully ${term2} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
+                        );
+                    }
+                }
+            }
+
+            if (this.enabled) {
+                device.meta.reporting = reportKey;
+            } else {
+                delete device.meta.reporting;
+                this.eventBus.emit('reportingDisabled', {device});
+            }
+
+            this.eventBus.emit(`devicesChanged`);
+        } catch (error) {
+            logger.error(
+                `Failed to ${term1.toLowerCase()} reporting for '${device.ieeeAddr}' - ${error.stack}`,
+            );
+
+            this.failed.add(device.ieeeAddr);
+        }
+
+        device.save();
+        this.queue.delete(device.ieeeAddr);
+    }
+
+    shouldSetupReporting(resolvedEntity, messageType) {
+        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
+            messageType === 'deviceLeave') return false;
+
+        const {device, definition} = resolvedEntity;
+        // Handle messages of type endDeviceAnnce and devIncoming.
+        // This message is typically send when a device comes online after being powered off
+        // Ikea TRADFRI tend to forget their reporting after powered off.
+        // Re-setup reporting.
+        // Only resetup reporting if configuredReportings was not populated yet,
+        // else reconfigure is done in zigbee-herdsman-converters ikea.js/bulbOnEvent
+        // configuredReportings are saved since Zigbee2MQTT 1.17.0
+        // https://github.com/Koenkk/zigbee2mqtt/issues/966
+        if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device) &&
+            device.endpoints.filter((e) => e.configuredReportings.length === 0).length === device.endpoints.length) {
+            return true;
+        }
+
+        // These do not support reproting.
+        // https://github.com/Koenkk/zigbee-herdsman/issues/110
+        const philipsIgnoreSw = ['5.127.1.26581', '5.130.1.30000'];
+        if (device.manufacturerName === 'Philips' && philipsIgnoreSw.includes(device.softwareBuildID)) return false;
+
+        if (resolvedEntity.device.interviewing === true) return false;
+        if (device.type !== 'Router' || device.powerSource === 'Battery') return false;
+        // Gledopto devices don't support reporting.
+        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
+
+        if (this.enabled && device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) {
+            return false;
+        }
+
+        if (!this.enabled && !device.meta.hasOwnProperty('reporting')) {
+            return false;
+        }
+
+        return true;
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByTypeLegacy('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClientsLegacy()) {
+            const resolvedEntity = this.zigbee.resolveEntityLegacy(device);
+            if (this.shouldSetupReporting(resolvedEntity, null)) {
+                await this.setupReporting(resolvedEntity);
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (this.shouldSetupReporting(resolvedEntity, type)) {
+            await this.setupReporting(resolvedEntity);
+        }
+    }
+}
+
+module.exports = Report;

lib/extension/legacy/softReset.js

@@ -0,0 +1,66 @@
+/* istanbul ignore file */
+// DEPRECATED
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
+
+/**
+ * This extensions soft resets the ZNP after a certain timeout.
+ */
+class SoftReset extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.timer = null;
+        this.timeout = utils.seconds(settings.get().advanced.soft_reset_timeout);
+    }
+
+    onZigbeeStarted() {
+        logger.debug(`Soft reset timeout set to ${this.timeout / 1000} seconds`);
+        this.resetTimer();
+    }
+
+    clearTimer() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+
+    resetTimer() {
+        if (this.timeout === 0) {
+            return;
+        }
+
+        this.clearTimer();
+        this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
+    }
+
+    async handleTimeout() {
+        logger.warn('Soft reset timeout triggered');
+
+        try {
+            await this.zigbee.reset('soft');
+            logger.warn('Soft resetted ZNP due to timeout');
+        } catch (error) {
+            logger.warn('Soft reset failed, trying stop/start');
+
+            await this.zigbee.stop();
+            logger.warn('Zigbee stopped');
+
+            try {
+                await this.zigbee.start();
+            } catch (error) {
+                logger.error('Failed to restart!');
+            }
+        }
+
+        this.resetTimer();
+    }
+
+    onZigbeeEvent() {
+        this.resetTimer();
+    }
+}
+
+module.exports = SoftReset;

lib/extension/livolo.js

@@ -1,183 +0,0 @@
-const utils = require('../util/utils');
-const interval = utils.secondsToMilliseconds(1);
-const logger = require('../util/logger');
-
-const foundationCfg = {manufSpec: 0, disDefaultRsp: 0};
-
-/**
- * Extension required for Livolo device support.
- */
-class Livolo {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-        this.configured = {};
-    }
-
-    onZigbeeStarted() {
-        this.startTimer();
-    }
-
-    _resetDeviceState(ieeeAddr) {
-        this.configured[ieeeAddr] = {
-            stage: 0,
-            retry: 0,
-            waitresp: false,
-        };
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (!device) {
-            return;
-        }
-
-        if ((message.type == 'devInterview') ||
-            (message.type == 'devIncoming') ||
-            (message.type == 'endDeviceAnnce') ||
-            (message.type == 'devStatus')) {
-            if (this.configured.hasOwnProperty(device.ieeeAddr)) {
-                logger.info(`LIVOLO ${device.ieeeAddr}. (Re)joins in the network (after power off?)`);
-                this._resetDeviceState(device.ieeeAddr);
-            }
-            return;
-        }
-    }
-
-    startTimer() {
-        this.clearTimer();
-        this.timer = setInterval(() => this.handleInterval(), interval);
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    stop() {
-        this.clearTimer();
-    }
-
-    _handleCommandRespSimple(err, rsp) {
-        this.ext.configured[this.ieeeAddr].waitresp = false;
-
-        if (err) {
-            if (this.ctype === 'toggle') {
-                logger.debug(`LIVOLO ${this.ieeeAddr}. Toggle command error:`, err.message);
-            }
-        } else {
-            logger.debug(`LIVOLO ${this.ieeeAddr}. Sucessfully configured`, rsp);
-            this.ext.configured[this.ieeeAddr].stage = 1; // sucessfully send command
-            this.ext.configured[this.ieeeAddr].retry = 0;
-            this.device.status = 'online';
-        }
-    }
-
-    _handleCommandRespWithData(err, rsp) {
-        this.ext.configured[this.ieeeAddr].waitresp = false;
-
-        if (err) {
-            logger.info(`LIVOLO ${this.ieeeAddr}. ${this.cid}.${this.ctype} response error:`, err.message);
-            if (this.ext.configured[this.ieeeAddr].retry >= 3) {
-                // errors in three sequental reads, stop polling, wait for a device message
-                this.device.status = 'offline';
-                logger.info(`LIVOLO ${this.ieeeAddr}. Stopped polling after 3 unsuccessful attempts`);
-            }
-        } else {
-            this.ext.configured[this.ieeeAddr].retry = 0;
-            this.device.status = 'online';
-            if (this.ext.zigbee) {
-                this.ext.zigbee.shepherd.emit('ind:reported', this.ep, this.cid, rsp, this.ep.last_af_msg);
-            }
-        }
-    }
-
-    // msg: { groupid, clusterid, srcaddr, srcendpoint, dstendpoint, wasbroadcast,
-    //   linkquality, securityuse, timestamp, transseqnumber, len, data }
-    _handleAfMessage(msg, ep) {
-        ep.linkquality = msg.linkquality;
-        ep.last_af_msg = msg;
-    }
-
-    _sendToggle(zdev, ieeeAddr, ep, retry) {
-        this.zigbee.queue.push(ieeeAddr, (queueCallback) => {
-            const cfg = {};
-            logger.debug(`LIVOLO ${ieeeAddr}. Sending the 'toggle' command. Retry: ${retry}`);
-            ep.functional('genOnOff', 'toggle', [cfg], foundationCfg,
-                this._handleCommandRespSimple.bind({
-                    device: zdev,
-                    ieeeAddr,
-                    cid: 'genOnOff',
-                    ctype: 'toggle',
-                    ext: this,
-                }));
-
-            queueCallback();
-        });
-    }
-
-    _sendPoll(zdev, ieeeAddr, ep, retry) {
-        this.zigbee.queue.push(ieeeAddr, (queueCallback) => {
-            ep.foundation('genOnOff', 'read', [{
-                attrId: 0, // onOff
-            }], this._handleCommandRespWithData.bind({
-                device: zdev,
-                ieeeAddr,
-                ep,
-                cid: 'genOnOff',
-                ctype: 'read',
-                ext: this,
-            }));
-
-            queueCallback();
-        });
-    }
-
-    handleInterval() {
-        this.zigbee.getAllClients()
-            .filter((d) => d.manufName && d.manufName.startsWith('LIVOLO')) // LIVOLO
-            .filter((d) => d.type === 'EndDevice') // Filter end devices
-            .filter((d) => d.powerSource && d.powerSource !== 'Battery') // Remove battery powered devices
-            .forEach((d) => {
-                const zdev = this.zigbee.shepherd._findDevByAddr(d.ieeeAddr);
-                if (zdev && zdev.endpoints) {
-                    const eplist = Object.keys(zdev.endpoints).filter((epId) => {
-                        const ep2 = zdev.getEndpoint(epId);
-                        const clist = ep2.getClusterList();
-                        return clist && clist.includes(6); // 6 - genOnOff
-                    });
-
-                    if (eplist.length > 0) {
-                        const ep = zdev.getEndpoint(eplist[0]);
-
-                        ep.onAfIncomingMsg = this._handleAfMessage;
-
-                        if (!this.configured.hasOwnProperty(d.ieeeAddr)) {
-                            this._resetDeviceState(d.ieeeAddr);
-                        }
-
-                        const state = this.configured[d.ieeeAddr];
-                        if (state.waitresp) {
-                            return;
-                        }
-
-                        if (state.retry < 3) {
-                            if (state.stage === 0) {
-                                state.retry += 1;
-                                state.waitresp = true;
-                                this._sendToggle(zdev, d.ieeeAddr, ep, state.retry);
-                            } else if (state.stage === 1) {
-                                state.retry += 1;
-                                state.waitresp = true;
-                                this._sendPoll(zdev, d.ieeeAddr, ep, state.retry);
-                            }
-                        }
-                    }
-                }
-            });
-        return true;
-    }
-}
-
-module.exports = Livolo;

lib/extension/networkMap.js

@@ -1,145 +0,0 @@
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-
-class NetworkMap {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.lastSeenMap = {};
-
-        // Subscribe to topic.
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
-        this.topicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
-
-        // Bind
-        this.raw = this.raw.bind(this);
-        this.graphviz = this.graphviz.bind(this);
-
-        // Set supported formats
-        this.supportedFormats = {
-            'raw': this.raw,
-            'graphviz': this.graphviz,
-        };
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (device) {
-            this.lastSeenMap[device.ieeeAddr] = Date.now();
-        }
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-        this.mqtt.subscribe(this.topicRoutes);
-    }
-
-    onMQTTMessage(topic, message) {
-        message = message.toString();
-        const includeRoutes = (topic === this.topicRoutes) ? 1: 0;
-        if ((topic === this.topic || topic === this.topicRoutes) &&
-            this.supportedFormats.hasOwnProperty(message)) {
-            this.zigbee.networkScan(includeRoutes, (result)=> {
-                const converted = this.supportedFormats[message](this.zigbee, result);
-                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
-            });
-
-            return true;
-        }
-
-        return false;
-    }
-
-    raw(zigbee, topology) {
-        return JSON.stringify(topology);
-    }
-
-    graphviz(zigbee, topology) {
-        const colors = settings.get().map_options.graphviz.colors;
-
-        let text = 'digraph G {\nnode[shape=record];\n';
-        let devStyle = '';
-
-        topology.nodes.forEach((device) => {
-            const labels = [];
-
-            // Add friendly name
-            labels.push(`${device.friendlyName}`);
-
-            // Add the device short network address and scan note (if any)
-            let scanNote = '';
-            if (device.scanfailed.includes('lqi')) {
-                scanNote += ' no lqi';
-            }
-            if (device.scanfailed.includes('rtg')) {
-                scanNote += ' no routes';
-            }
-            labels.push(`0x${device.nwkAddr.toString(16)} ${scanNote}`);
-
-            // Add the device model
-            const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-            if (mappedModel) {
-                labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
-            } else {
-                // This model is not supported by zigbee-shepherd-converters, add zigbee model information, if available
-                const zigbeeModel = [device.manufName, device.modelId].filter((a) => a).join(' ');
-                labels.push(zigbeeModel ? zigbeeModel : 'No model information available');
-            }
-
-            // Add the device status (online/offline) and last_seen timestamp
-            let lastSeen = 'unknown';
-            if (device.type == 'Coordinator') {
-                lastSeen = utils.formatDate(Date.now(), settings.get().advanced.last_seen,
-                    utils.toLocalISOString(new Date(Date.now())));
-            } else {
-                if (this.lastSeenMap[device.ieeeAddr]) {
-                    const lastSeenAgo =
-                        `${new Date(Date.now() - this.lastSeenMap[device.ieeeAddr]).toISOString().substr(11, 8)}s ago`;
-                    lastSeen = utils.formatDate(this.lastSeenMap[device.ieeeAddr],
-                        settings.get().advanced.last_seen, lastSeenAgo);
-                }
-            }
-            labels.push(`${device.status} (${lastSeen})`);
-
-            // Shape the record according to device type
-            if (device.type == 'Coordinator') {
-                devStyle = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
-                    `fontcolor="${colors.font.coordinator}"`;
-            } else if (device.type == 'Router') {
-                devStyle = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
-                    `fontcolor="${colors.font.router}"`;
-            } else {
-                devStyle = `style="rounded, dashed, filled", fillcolor="${colors.fill.enddevice}", ` +
-                    `fontcolor="${colors.font.enddevice}"`;
-            }
-
-            // Add the device with its labels to the graph as a node.
-            text += `  "${device.ieeeAddr}" [`+devStyle+`, label="{${labels.join('|')}}"];\n`;
-
-            /**
-             * Add an edge between the device and its child to the graph
-             * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
-             * due to not responded to the lqi scan. In that case we do not add an edge for this device.
-             */
-            topology.links.filter((e) => (e.sourceIeeeAddr === device.ieeeAddr) || (e.SourceNwkAddr === device.nwkAddr))
-                .forEach((e) => {
-                    const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, ' : (!e.routes.length) ?
-                        'penwidth=0.5, ' : 'penwidth=2, ';
-                    const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
-                        `weight=1, color="${colors.line.active}", `;
-                    const textRoutes = e.routes.map((r) => `0x${r.toString(16)}`);
-                    const lineLabels = (!e.routes.length) ? `label="${e.lqi}"` :
-                        `label="${e.lqi}\\n[${textRoutes.join(']\\n[')}]"`;
-                    text += `  "${device.ieeeAddr}" -> "${e.targetIeeeAddr}"`;
-                    text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
-                });
-        });
-
-        text += '}';
-
-        return text.replace(/\0/g, '');
-    }
-}
-
-module.exports = NetworkMap;

lib/extension/networkMap.ts

@@ -0,0 +1,324 @@
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import logger from '../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+
+interface Link {
+    source: {ieeeAddr: string, networkAddress: number}, target: {ieeeAddr: string, networkAddress: number},
+    linkquality: number, depth: number, routes: RoutingTableEntry[],
+    sourceIeeeAddr: string, targetIeeeAddr: string, sourceNwkAddr: number, lqi: number, relationship: number,
+}
+
+interface Topology {
+    nodes: {
+        ieeeAddr: string, friendlyName: string, type: string, networkAddress: number, manufacturerName: string,
+        modelID: string, failed: string[], lastSeen: number,
+        definition: {model: string, vendor: string, supports: string, description: string}}[],
+    links: Link[],
+}
+
+/**
+ * This extension creates a network map
+ */
+class NetworkMap extends ExtensionTS {
+    private legacyApi = settings.get().advanced.legacy_api;
+    private legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+    private legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+    private topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
+    private supportedFormats: {[s: string]: (topology: Topology) => KeyValue | string};
+
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+        this.supportedFormats = {
+            'raw': this.raw,
+            'graphviz': this.graphviz,
+            'plantuml': this.plantuml,
+        };
+    }
+
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if ((data.topic === this.legacyTopic || data.topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(data.message)) {
+                const includeRoutes = data.topic === this.legacyTopicRoutes;
+                const topology = await this.networkScan(includeRoutes);
+                let converted = this.supportedFormats[data.message](topology);
+                converted = data.message === 'raw' ? stringify(converted) : converted;
+                this.mqtt.publish(`bridge/networkmap/${data.message}`, converted as string, {});
+            }
+        }
+
+        if (data.topic === this.topic) {
+            const message = utils.parseJSON(data.message, data.message);
+            try {
+                const type = typeof message === 'object' ? message.type : message;
+                if (!this.supportedFormats.hasOwnProperty(type)) {
+                    throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
+                }
+
+                const routes = typeof message === 'object' && message.routes;
+                const topology = await this.networkScan(routes);
+                const value = this.supportedFormats[type](topology);
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {routes, type, value}, null)),
+                );
+            } catch (error) {
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {}, error.message)),
+                );
+            }
+        }
+    }
+
+    @bind raw(topology: Topology): KeyValue {
+        return topology;
+    }
+
+    @bind graphviz(topology: Topology): string {
+        const colors = settings.get().map_options.graphviz.colors;
+
+        let text = 'digraph G {\nnode[shape=record];\n';
+        let style = '';
+
+        topology.nodes.forEach((node) => {
+            const labels = [];
+
+            // Add friendly name
+            labels.push(`${node.friendlyName}`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            labels.push(
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
+                ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                if (node.definition) {
+                    labels.push(`${node.definition.vendor} ${node.definition.description} (${node.definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    labels.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative') as string;
+            }
+
+            labels.push(lastSeen);
+
+            // Shape the record according to device type
+            if (node.type == 'Coordinator') {
+                style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
+                    `fontcolor="${colors.font.coordinator}"`;
+            } else if (node.type == 'Router') {
+                style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
+                    `fontcolor="${colors.font.router}"`;
+            } else {
+                style = `style="rounded, dashed, filled", fillcolor="${colors.fill.enddevice}", ` +
+                    `fontcolor="${colors.font.enddevice}"`;
+            }
+
+            // Add the device with its labels to the graph as a node.
+            text += `  "${node.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
+
+            /**
+             * Add an edge between the device and its child to the graph
+             * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+             * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+             */
+            topology.links.filter((e) => (e.source.ieeeAddr === node.ieeeAddr)).forEach((e) => {
+                const lineStyle = (node.type=='EndDevice') ? 'penwidth=1, ' :
+                    (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
+                const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
+                    `weight=1, color="${colors.line.active}", `;
+                const textRoutes = e.routes.map((r) => utils.toNetworkAddressHex(r.destinationAddress));
+                const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
+                    `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
+                text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
+            });
+        });
+
+        text += '}';
+
+        return text.replace(/\0/g, '');
+    }
+
+    @bind plantuml(topology: Topology): string {
+        const text = [];
+
+        text.push(`' paste into: https://www.planttext.com/`);
+        text.push(``);
+        text.push('@startuml');
+
+        topology.nodes.sort((a, b) => a.friendlyName.localeCompare(b.friendlyName)).forEach((node) => {
+            // Add friendly name
+            text.push(`card ${node.ieeeAddr} [`);
+            text.push(`${node.friendlyName}`);
+            text.push(`---`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            text.push(
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
+                ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                text.push(`---`);
+                const definition = this.zigbee.resolveEntityLegacy(node.ieeeAddr).definition;
+                if (definition) {
+                    text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    text.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative') as string;
+            }
+            text.push(`---`);
+            text.push(lastSeen);
+            text.push(`]`);
+            text.push(``);
+        });
+
+        /**
+         * Add edges between the devices
+         * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+         * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+         */
+        topology.links.forEach((link) => {
+            text.push(`${link.sourceIeeeAddr} --> ${link.targetIeeeAddr}: ${link.lqi}`);
+        });
+        text.push('');
+
+        text.push(`@enduml`);
+
+        return text.join(`\n`);
+    }
+
+    async networkScan(includeRoutes: boolean): Promise<Topology> {
+        logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
+        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
+        const lqis: Map<Device, LQI> = new Map();
+        const routingTables: Map<Device, RoutingTable> = new Map();
+        const failed: Map<Device, string[]> = new Map();
+
+        for (const device of devices.filter((d) => d.type != 'EndDevice')) {
+            failed.set(device, []);
+            await utils.sleep(1); // sleep 1 second between each scan to reduce stress on network.
+
+            const doRequest = async <T>(request: () => Promise<T>, firstAttempt = true): Promise<T> => {
+                try {
+                    return await request();
+                } catch (error) {
+                    if (!firstAttempt) {
+                        throw error;
+                    } else {
+                        // Network is possibly congested, sleep 5 seconds to let the network settle.
+                        await utils.sleep(5);
+                        return doRequest(request, false);
+                    }
+                }
+            };
+
+            try {
+                const result = await doRequest<LQI>(async () => device.lqi());
+                lqis.set(device, result);
+                logger.debug(`LQI succeeded for '${device.name}'`);
+            } catch (error) {
+                failed.get(device).push('lqi');
+                logger.error(`Failed to execute LQI for '${device.name}'`);
+            }
+
+            if (includeRoutes) {
+                try {
+                    const result = await doRequest(async () => device.routingTable());
+                    routingTables.set(device, result);
+                    logger.debug(`Routing table succeeded for '${device.name}'`);
+                } catch (error) {
+                    failed.get(device).push('routingTable');
+                    logger.error(`Failed to execute routing table for '${device.name}'`);
+                }
+            }
+        }
+
+        logger.info(`Network scan finished`);
+
+        const topology: Topology = {nodes: [], links: []};
+        // Add nodes
+        for (const device of devices) {
+            const definition = device.definition ? {
+                model: device.definition.model,
+                vendor: device.definition.vendor,
+                description: device.definition.description,
+                supports: Array.from(new Set((device.definition.exposes).map((e) => {
+                    return e.hasOwnProperty('name') ? e.name :
+                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
+                }))).join(', '),
+            } : null;
+
+            topology.nodes.push({
+                ieeeAddr: device.ieeeAddr, friendlyName: device.name, type: device.type,
+                networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
+                modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+                definition,
+            });
+        }
+
+        // Add links
+        lqis.forEach((lqi, device) => {
+            for (const neighbor of lqi.neighbors) {
+                if (neighbor.relationship > 3) {
+                    // Relationship is not active, skip it
+                    continue;
+                }
+
+                // Some Xiaomi devices return 0x00 as the neighbor ieeeAddr (obviously not correct).
+                // Determine the correct ieeeAddr based on the networkAddress.
+                const neighborDevice = this.zigbee.deviceByNetworkAddress(neighbor.networkAddress);
+                if (neighbor.ieeeAddr === '0x0000000000000000' && neighborDevice) {
+                    neighbor.ieeeAddr = neighborDevice.ieeeAddr;
+                }
+
+                const link: Link = {
+                    source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
+                    target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},
+                    linkquality: neighbor.linkquality, depth: neighbor.depth, routes: [],
+                    // DEPRECATED:
+                    sourceIeeeAddr: neighbor.ieeeAddr, targetIeeeAddr: device.ieeeAddr,
+                    sourceNwkAddr: neighbor.networkAddress, lqi: neighbor.linkquality,
+                    relationship: neighbor.relationship,
+                };
+
+                const routingTable = routingTables.get(device);
+                if (routingTable) {
+                    link.routes = routingTable.table
+                        .filter((t) => t.status === 'ACTIVE' && t.nextHop === neighbor.networkAddress);
+                }
+
+                topology.links.push(link);
+            }
+        });
+
+        return topology;
+    }
+}
+
+module.exports = NetworkMap;

lib/extension/onEvent.ts

@@ -0,0 +1,45 @@
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import ExtensionTS from './extensionts';
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends ExtensionTS {
+    override async start(): Promise<void> {
+        for (const device of this.zigbee.getClients()) {
+            await this.callOnEvent(device, 'start', {});
+        }
+
+        this.eventBus.onDeviceMessage(this, (data) => this.callOnEvent(data.device, 'message', this.convertData(data)));
+        this.eventBus.onDeviceJoined(this,
+            (data) => this.callOnEvent(data.device, 'deviceJoined', this.convertData(data)));
+        this.eventBus.onDeviceInterview(this,
+            (data) => this.callOnEvent(data.device, 'deviceInterview', this.convertData(data)));
+        this.eventBus.onDeviceAnnounce(this,
+            (data) => this.callOnEvent(data.device, 'deviceAnnounce', this.convertData(data)));
+        this.eventBus.onDeviceNetworkAddressChanged(this,
+            (data) => this.callOnEvent(data.device, 'deviceNetworkAddressChanged', this.convertData(data)));
+    }
+
+    private convertData(data: KeyValue): KeyValue {
+        return {...data, device: data.device.zhDevice};
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        for (const device of this.zigbee.getClients()) {
+            await this.callOnEvent(device, 'stop', {});
+        }
+    }
+
+    private async callOnEvent(device: Device, type: string, data: KeyValue): Promise<void> {
+        zhc.onEvent(type, data, device.zhDevice, device.settings);
+
+        if (device.definition?.onEvent) {
+            await device.definition.onEvent(type, data, device.zhDevice, device.settings);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.ts

@@ -0,0 +1,281 @@
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import * as utils from '../util/utils';
+// @ts-ignore
+import tradfriOTA from 'zigbee-herdsman-converters/lib/ota/tradfri';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+import Device from '../model/device';
+
+type UpdateState = 'updating' | 'idle' | 'available';
+interface UpdatePayload {
+    // eslint-disable-next-line camelcase
+    update: {progress?: number, remaining?: number, state: UpdateState}, update_available?: boolean}
+
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
+
+class OTAUpdate extends ExtensionTS {
+    private inProgress = new Set();
+    private lastChecked: {[s: string]: number} = {};
+    private legacyApi = settings.get().advanced.legacy_api;
+
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+        this.eventBus.onDeviceMessage(this, this.onZigbeeEvent);
+        if (settings.get().advanced.ikea_ota_use_test_url) {
+            tradfriOTA.useTestURL();
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
+            // remove them.
+            this.removeProgressAndRemainingFromState(device);
+        }
+    }
+
+    private removeProgressAndRemainingFromState(device: Device): void {
+        this.state.removeKey(device.ieeeAddr, ['update', 'progress']);
+        this.state.removeKey(device.ieeeAddr, ['update', 'remaining']);
+    }
+
+    @bind private async onZigbeeEvent(data: EventDeviceMessage): Promise<void> {
+        if (settings.get().ota.disable_automatic_update_check) return;
+        if (data.type !== 'commandQueryNextImageRequest' || !data.device.definition) return;
+        logger.debug(`Device '${data.device.name}' requested OTA`);
+
+        const supportsOTA = data.device.definition.hasOwnProperty('ota');
+        if (supportsOTA) {
+            // When a device does a next image request, it will usually do it a few times after each other
+            // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
+            // each time, so this interval can be set by the user. The default is 1,440 minutes (one day).
+            const updateCheckInterval = settings.get().ota.update_check_interval * 1000 * 60;
+            const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > updateCheckInterval : true;
+            if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
+
+            this.lastChecked[data.device.ieeeAddr] = Date.now();
+            const available = await data.device.definition.ota.isUpdateAvailable(
+                data.device.zhDevice, logger, data.data);
+            const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+            this.publishEntityState(data.device.ieeeAddr, payload);
+
+            if (available) {
+                const message = `Update available for '${data.device.name}'`;
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: 'available', device: data.device.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        }
+
+        // Respond to the OTA request:
+        // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
+        // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
+        const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
+        if (endpoint) {
+            // Some devices send OTA requests without defining OTA cluster as input cluster.
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        }
+    }
+
+    private async readSoftwareBuildIDAndDateCode(device: Device, update: boolean):
+        Promise<{softwareBuildID: string, dateCode: string}> {
+        try {
+            const endpoint = device.endpoints.find((e) => e.supportsInputCluster('genBasic'));
+            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
+
+            if (update) {
+                device.zhDevice.softwareBuildID = result.swBuildId;
+                device.zhDevice.dateCode = result.dateCode;
+                device.zhDevice.save();
+            }
+
+            return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
+        } catch (e) {
+            return null;
+        }
+    }
+
+    private getEntityPublishPayload(state: UpdateState, progress: number=null, remaining: number=null): UpdatePayload {
+        const payload: UpdatePayload = {update: {state}};
+        if (progress !== null) payload.update.progress = progress;
+        if (remaining !== null) payload.update.remaining = Math.round(remaining);
+
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            payload.update_available = state === 'available';
+        }
+
+        return payload;
+    }
+
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        if ((!this.legacyApi || !data.topic.match(legacyTopicRegex)) && !data.topic.match(topicRegex)) {
+            return null;
+        }
+
+        const message = utils.parseJSON(data.message, data.message);
+        const ID = (typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message) as string;
+        const device = this.zigbee.resolveEntity(ID);
+        const type = data.topic.substring(data.topic.lastIndexOf('/') + 1);
+        const responseData: {id: string, updateAvailable?: boolean, from?: string, to?: string}= {id: ID};
+        let error = null;
+        let errorStack = null;
+
+        if (!(device instanceof Device)) {
+            error = `Device '${ID}' does not exist`;
+        } else if (!device.definition || !device.definition.ota) {
+            error = `Device '${device.name}' does not support OTA updates`;
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `not_supported`, device: device.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    stringify({type: `ota_update`, message: error, meta}),
+                );
+            }
+        } else if (this.inProgress.has(device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${device.name}'`;
+        } else {
+            this.inProgress.add(device.ieeeAddr);
+
+            if (type === 'check') {
+                const msg = `Checking if update available for '${device.name}'`;
+                logger.info(msg);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `checking_if_available`, device: device.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, message: msg, meta}),
+                    );
+                }
+
+                try {
+                    const available = await device.definition.ota.isUpdateAvailable(device.zhDevice, logger);
+                    const msg = `${available ? 'Update' : 'No update'} available for '${device.name}'`;
+                    logger.info(msg);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: available ? 'available' : 'not_available', device: device.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: msg, meta}),
+                        );
+                    }
+
+                    const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+                    this.publishEntityState(device.ieeeAddr, payload);
+                    this.lastChecked[device.ieeeAddr] = Date.now();
+                    responseData.updateAvailable = available;
+                } catch (e) {
+                    error = `Failed to check if update available for '${device.name}' (${e.message})`;
+                    errorStack = e.stack;
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `check_failed`, device: device.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: error, meta}),
+                        );
+                    }
+                }
+            } else { // type === 'update'
+                const msg = `Updating '${device.name}' to latest firmware`;
+                logger.info(msg);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_in_progress`, device: device.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, message: msg, meta}),
+                    );
+                }
+
+                try {
+                    const onProgress = (progress: number, remaining: number): void => {
+                        let msg = `Update of '${device.name}' at ${progress.toFixed(2)}%`;
+                        if (remaining) {
+                            msg += `, ≈ ${Math.round(remaining / 60)} minutes remaining`;
+                        }
+
+                        logger.info(msg);
+
+                        const payload = this.getEntityPublishPayload('updating', progress, remaining);
+                        this.publishEntityState(device.ieeeAddr, payload);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const meta = {status: `update_progress`, device: device.name, progress};
+                            this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: msg, meta}));
+                        }
+                    };
+
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(device, false);
+                    await device.definition.ota.updateToLatest(device.zhDevice, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(device, true);
+                    const [fromS, toS] = [stringify(from_), stringify(to)];
+                    const msg = `Finished update of '${device.name}'` +
+                        (to ? `, from '${fromS}' to '${toS}'` : ``);
+                    logger.info(msg);
+                    this.removeProgressAndRemainingFromState(device);
+                    const payload = this.getEntityPublishPayload('idle');
+                    this.publishEntityState(device.ieeeAddr, payload);
+                    responseData.from = from_ ? utils.toSnakeCase(from_) : null;
+                    responseData.to = to ? utils.toSnakeCase(to) : null;
+                    this.eventBus.emit(`devicesChanged`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_succeeded`, device: device.name, from: from_, to};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
+                    }
+                } catch (e) {
+                    logger.debug(`Update of '${device.name}' failed (${e})`);
+                    error = `Update of '${device.name}' failed (${e.message})`;
+                    errorStack = e.stack;
+
+                    this.removeProgressAndRemainingFromState(device);
+                    const payload = this.getEntityPublishPayload('available');
+                    this.publishEntityState(device.ieeeAddr, payload);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_failed`, device: device.name};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: error, meta}));
+                    }
+                }
+            }
+
+            this.inProgress.delete(device.ieeeAddr);
+        }
+
+        const triggeredViaLegacyApi = data.topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+            errorStack && logger.debug(errorStack);
+        }
+    }
+}
+
+module.exports = OTAUpdate;

lib/extension/publish.ts

@@ -0,0 +1,289 @@
+
+import * as settings from '../util/settings';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import logger from '../util/logger';
+import * as utils from '../util/utils';
+import ExtensionTS from './extensionts';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import Group from '../model/group';
+import Device from '../model/device';
+import bind from 'bind-decorator';
+
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.endpointNames.join('|')}))?/(get|set)(?:/(.+))?`);
+const propertyEndpointRegex = new RegExp(`^(.*)_(${utils.endpointNames.join('|')})$`);
+const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
+
+// Legacy: don't provide default converters anymore, this is required by older z2m installs not saving group members
+const defaultGroupConverters = [
+    zhc.toZigbeeConverters.light_onoff_brightness,
+    zhc.toZigbeeConverters.light_color_colortemp,
+    zhc.toZigbeeConverters.effect,
+    zhc.toZigbeeConverters.ignore_transition,
+    zhc.toZigbeeConverters.cover_position_tilt,
+    zhc.toZigbeeConverters.thermostat_occupied_heating_setpoint,
+    zhc.toZigbeeConverters.tint_scene,
+    zhc.toZigbeeConverters.light_brightness_move,
+    zhc.toZigbeeConverters.light_brightness_step,
+    zhc.toZigbeeConverters.light_colortemp_step,
+    zhc.toZigbeeConverters.light_colortemp_move,
+    zhc.toZigbeeConverters.light_hue_saturation_move,
+    zhc.toZigbeeConverters.light_hue_saturation_step,
+];
+
+interface ParsedTopic {ID: string, endpoint: string, attribute: string, type: 'get' | 'set'}
+
+class Publish extends ExtensionTS {
+    async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+    }
+
+    parseTopic(topic: string): ParsedTopic | null {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
+        // If we didn't replace base_topic we received something we don't care about
+        if (ID === match[1] || ID.match(/bridge/)) {
+            return null;
+        }
+
+        return {ID: ID, endpoint: match[2], type: match[3] as 'get' | 'set', attribute: match[4]};
+    }
+
+    parseMessage(parsedTopic: ParsedTopic, data: EventMQTTMessage): KeyValue | null {
+        if (parsedTopic.attribute) {
+            try {
+                return {[parsedTopic.attribute]: JSON.parse(data.message)};
+            } catch (e) {
+                return {[parsedTopic.attribute]: data.message};
+            }
+        } else {
+            try {
+                return JSON.parse(data.message);
+            } catch (e) {
+                if (stateValues.includes(data.message.toLowerCase())) {
+                    return {state: data.message};
+                } else {
+                    return null;
+                }
+            }
+        }
+    }
+
+    legacyLog(payload: KeyValue): void {
+        /* istanbul ignore else */
+        if (settings.get().advanced.legacy_api) {
+            this.mqtt.publish('bridge/log', stringify(payload));
+        }
+    }
+
+    legacyRetrieveState(re: Device | Group, converter: ToZigbeeConverter, result: ToZigbeeConverterResult,
+        target: ZHEndpoint | ZHGroup, key: string, meta: ToZigbeeConverterGetMeta): void {
+        // It's possible for devices to get out of sync when writing an attribute that's not reportable.
+        // So here we re-read the value after a specified timeout, this timeout could for example be the
+        // transition time of a color change or for forcing a state read for devices that don't
+        // automatically report a new state when set.
+        // When reporting is requested for a device (report: true in device-specific settings) we won't
+        // ever issue a read here, as we assume the device will properly report changes.
+        // Only do this when the retrieve_state option is enabled for this device.
+        // retrieve_state == decprecated
+        if (re instanceof Device && result && result.hasOwnProperty('readAfterWriteTime') &&
+            re.settings.retrieve_state
+        ) {
+            setTimeout(() => converter.convertGet(target, key, meta), result.readAfterWriteTime);
+        }
+    }
+
+    updateMessageHomeAssistant(message: KeyValue, entityState: KeyValue): void {
+        /**
+         * Home Assistant always publishes 'state', even when e.g. only setting
+         * the color temperature. This would lead to 2 zigbee publishes, where the first one
+         * (state) is probably unecessary.
+         */
+        if (settings.get().homeassistant) {
+            const hasColorTemp = message.hasOwnProperty('color_temp');
+            const hasColor = message.hasOwnProperty('color');
+            const hasBrightness = message.hasOwnProperty('brightness');
+            const isOn = entityState.state === 'ON' ? true : false;
+            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
+                delete message.state;
+                logger.debug('Skipping state because of Home Assistant');
+            }
+        }
+    }
+
+    // TODO remove trailing _
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        const parsedTopic = this.parseTopic(data.topic);
+        if (!parsedTopic) return;
+
+        const re = this.zigbee.resolveEntity(parsedTopic.ID);
+        if (re == null) {
+            this.legacyLog({type: `entity_not_found`, message: {friendly_name: parsedTopic.ID}});
+            logger.error(`Entity '${parsedTopic.ID}' is unknown`);
+            return;
+        }
+
+        // Get entity details
+        const definition = re instanceof Device ? re.definition : re.membersDefinitions();
+        if (definition == null) {
+            logger.error(`Cannot publish to unsupported device '${re.name}'`);
+            return;
+        }
+        const target = re instanceof Group ? re.zhGroup : re.endpoint(parsedTopic.endpoint);
+        if (target == null) {
+            logger.error(`Device '${re.name}' has no endpoint '${parsedTopic.endpoint}'`);
+            return;
+        }
+        const device = re instanceof Device ? re.zhDevice : null;
+        const entitySettings = re.settings;
+        const entityState = this.state.get(re.ID) || {};
+        const membersState = re instanceof Group ?
+            Object.fromEntries(re.membersIeeeAddr().map((e) => [e, this.state.get(e)])) : null;
+        let converters: ToZigbeeConverter[];
+        {
+            if (Array.isArray(definition)) {
+                const c = new Set(definition.map((d) => d.toZigbee).flat());
+                if (c.size == 0) converters = defaultGroupConverters;
+                else converters = Array.from(c);
+            } else {
+                converters = definition.toZigbee;
+            }
+        }
+
+        // Convert the MQTT message to a Zigbee message.
+        const message = this.parseMessage(parsedTopic, data);
+        if (message == null) {
+            logger.error(`Invalid message '${message}', skipping...`);
+            return;
+        }
+        this.updateMessageHomeAssistant(message, entityState);
+
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
+        const entries = Object.entries(message);
+        const sorter = typeof message.state === 'string' && message.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
+
+        // For each attribute call the corresponding converter
+        const usedConverters: {[s: number]: ToZigbeeConverter[]} = {};
+        const toPublish: {[s: number | string]: KeyValue} = {};
+        const addToToPublish = (ID: number | string, payload: KeyValue): void => {
+            if (!(ID in toPublish)) toPublish[ID] = {};
+            toPublish[ID] = {...toPublish[ID], ...payload};
+        };
+
+        for (let [key, value] of entries) {
+            let endpointName = parsedTopic.endpoint;
+            let localTarget = target;
+            // @ts-ignore TODO fix
+            let endpointOrGroupID = target.constructor.name === 'Endpoint' ? target.ID : target.groupID;
+
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            const propertyEndpointMatch = key.match(propertyEndpointRegex);
+            if (re instanceof Device && propertyEndpointMatch) {
+                endpointName = propertyEndpointMatch[2];
+                key = propertyEndpointMatch[1];
+                localTarget = re.endpoint(endpointName);
+                if (localTarget == null) {
+                    logger.error(`Device '${re.name}' has no endpoint '${endpointName}'`);
+                    continue;
+                }
+                endpointOrGroupID = localTarget.ID;
+            }
+
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
+            const converter = converters.find((c) => c.key.includes(key));
+
+            if (parsedTopic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter for set only once
+                // (e.g. light_onoff_brightness converters can convert state and brightness)
+                continue;
+            }
+
+            if (!converter) {
+                logger.error(`No converter available for '${key}' (${message[key]})`);
+                continue;
+            }
+
+            // Converter didn't return a result, skip
+            const meta = {endpoint_name: endpointName, options: entitySettings, message: {...message}, logger, device,
+                state: entityState, membersState, mapped: definition};
+
+            // Strip endpoint name from meta.message properties.
+            if (endpointName) {
+                for (const [key, value] of Object.entries(meta.message)) {
+                    if (key.endsWith(endpointName)) {
+                        delete meta.message[key];
+                        const keyWithoutEndpoint = key.substring(0, key.length - endpointName.length - 1);
+                        meta.message[keyWithoutEndpoint] = value;
+                    }
+                }
+            }
+
+            try {
+                if (parsedTopic.type === 'set' && converter.convertSet) {
+                    logger.debug(`Publishing '${parsedTopic.type}' '${key}' to '${re.name}'`);
+                    const result = await converter.convertSet(localTarget, key, value, meta);
+                    const optimistic = !entitySettings.hasOwnProperty('optimistic') || entitySettings.optimistic;
+                    if (result && result.state && optimistic) {
+                        const msg = result.state;
+
+                        if (endpointName) {
+                            for (const key of Object.keys(msg)) {
+                                msg[`${key}_${endpointName}`] = msg[key];
+                                delete msg[key];
+                            }
+                        }
+
+                        // filter out attribute listed in filtered_optimistic
+                        entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
+                        addToToPublish(re.ID, msg);
+                    }
+
+                    if (result && result.membersState && optimistic) {
+                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
+                            addToToPublish(ieeeAddr, state);
+                        }
+                    }
+
+                    this.legacyRetrieveState(re, converter, result, localTarget, key, meta);
+                } else if (parsedTopic.type === 'get' && converter.convertGet) {
+                    logger.debug(`Publishing get '${parsedTopic.type}' '${key}' to '${re.name}'`);
+                    await converter.convertGet(localTarget, key, meta);
+                } else {
+                    logger.error(`No converter available for '${parsedTopic.type}' '${key}' (${message[key]})`);
+                    continue;
+                }
+            } catch (error) {
+                const message =
+                    `Publish '${parsedTopic.type}' '${key}' to '${re.name}' failed: '${error}'`;
+                logger.error(message);
+                logger.debug(error.stack);
+                this.legacyLog({type: `zigbee_publish_error`, message, meta: {friendly_name: re.name}});
+            }
+
+            usedConverters[endpointOrGroupID].push(converter);
+        }
+
+        for (const [ID, payload] of Object.entries(toPublish)) {
+            if (Object.keys(payload).length != 0) {
+                this.publishEntityState(ID, payload);
+            }
+        }
+    }
+}
+
+// TODO_finished: : change class to export default
+module.exports = Publish;

lib/extension/receive.ts

@@ -0,0 +1,162 @@
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+import debounce from 'debounce';
+import ExtensionTS from './extensionts';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import bind from 'bind-decorator';
+
+class Receive extends ExtensionTS {
+    private elapsed: {[s: string]: number} = {};
+    // eslint-disable-next-line
+    private debouncers: {[s: string]: {payload: KeyValue, publish: any}} = {}; //TODO fix type
+
+    async start(): Promise<void> {
+        this.eventBus.onPublishEntityState(this, this.onPublishEntityState);
+        this.eventBus.onDeviceMessage(this, this.onDeviceMessage);
+    }
+
+    @bind async onPublishEntityState(data: EventPublishEntityState): Promise<void> {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.ieeeAddr && this.debouncers[data.ieeeAddr] && data.stateChangeReason !== 'publishDebounce') {
+            for (const key of Object.keys(data.payload)) {
+                delete this.debouncers[data.ieeeAddr].payload[key];
+            }
+        }
+    }
+
+    publishDebounce(ieeeAddr: string, payload: KeyValue, time: number, debounceIgnore: string[]): void {
+        if (!this.debouncers[ieeeAddr]) {
+            this.debouncers[ieeeAddr] = {
+                payload: {},
+                publish: debounce(() => {
+                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
+                    this.debouncers[ieeeAddr].payload = {};
+                }, time * 1000),
+            };
+        }
+
+        if (this.isPayloadConflicted(payload, this.debouncers[ieeeAddr].payload, debounceIgnore)) {
+            // publish previous payload immediately
+            this.debouncers[ieeeAddr].publish.flush();
+        }
+
+        // extend debounced payload with current
+        this.debouncers[ieeeAddr].payload = {...this.debouncers[ieeeAddr].payload, ...payload};
+        this.debouncers[ieeeAddr].publish();
+    }
+
+    // if debounce_ignore are specified (Array of strings)
+    // then all newPayload values with key present in debounce_ignore
+    // should equal or be undefined in oldPayload
+    // otherwise payload is conflicted
+    isPayloadConflicted(newPayload: KeyValue, oldPayload: KeyValue, debounceIgnore: string[] | null): boolean {
+        let result = false;
+        Object.keys(oldPayload)
+            .filter((key) => (debounceIgnore || []).includes(key))
+            .forEach((key) => {
+                if (typeof newPayload[key] !== 'undefined' && newPayload[key] !== oldPayload[key]) {
+                    result = true;
+                }
+            });
+
+        return result;
+    }
+
+    shouldProcess(data: EventDeviceMessage): boolean {
+        if (!data.device.definition) {
+            if (data.device.interviewing) {
+                logger.debug(`Skipping message, definition is undefined and still interviewing`);
+            } else {
+                logger.warn(
+                    `Received message from unsupported device with Zigbee model '${data.device.modelID}' ` +
+                    `and manufacturer name '${data.device.manufacturerName}'`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            }
+
+            return false;
+        }
+
+        return true;
+    }
+
+    @bind onDeviceMessage(data: EventDeviceMessage): void {
+        /* istanbul ignore next */
+        if (!data.device) return;
+
+        /**
+         * Handling of re-transmitted Xiaomi messages.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
+         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
+         *
+         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
+         * The network address of these message is set to the one of the Xiaomi router.
+         * Therefore it looks like if the message came from the Xiaomi router, while in
+         * fact it came from the end device.
+         * Handling these message would result in false state updates.
+         * The group ID attribute of these message defines the network address of the end device.
+         */
+        if (data.device.isXiaomiDevice() && data.device.isRouter() && data.groupID) {
+            logger.debug('Handling re-transmitted Xiaomi message');
+            data = {...data, device: this.zigbee.deviceByNetworkAddress(data.groupID)};
+        }
+
+        if (!this.shouldProcess(data)) return;
+
+        const converters = data.device.definition.fromZigbee.filter((c) => {
+            const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
+            return c.cluster === data.cluster && type;
+        });
+
+        // Check if there is an available converter, genOta messages are not interesting.
+        const ignoreClusters: (string | number)[] = ['genOta', 'genTime', 'genBasic'];
+        if (converters.length == 0 && !ignoreClusters.includes(data.cluster)) {
+            logger.debug(`No converter available for '${data.device.definition.model}' with ` +
+                `cluster '${data.cluster}' and type '${data.type}' and data '${stringify(data.data)}'`);
+            return;
+        }
+
+        // Convert this Zigbee message to a MQTT message.
+        // Get payload for the message.
+        // - If a payload is returned publish it to the MQTT broker
+        // - If NO payload is returned do nothing. This is for non-standard behaviour
+        //   for e.g. click switches where we need to count number of clicks and detect long presses.
+        const publish = (payload: KeyValue): void => {
+            if (settings.get().advanced.elapsed) {
+                const now = Date.now();
+                if (this.elapsed[data.device.ieeeAddr]) {
+                    payload.elapsed = now - this.elapsed[data.device.ieeeAddr];
+                }
+
+                this.elapsed[data.device.ieeeAddr] = now;
+            }
+
+            // Check if we have to debounce
+            if (data.device.settings.debounce) {
+                this.publishDebounce(data.device.ieeeAddr, payload, data.device.settings.debounce,
+                    data.device.settings.debounce_ignore);
+            } else {
+                this.publishEntityState(data.device.ieeeAddr, payload);
+            }
+        };
+
+        const meta = {device: data.device.zhDevice, logger, state: this.state.get(data.device.ieeeAddr)};
+        let payload = {};
+        converters.forEach((converter) => {
+            const converted = converter.convert(data.device.definition, data, publish, data.device.settings, meta);
+            if (converted) {
+                payload = {...payload, ...converted};
+            }
+        });
+
+        if (Object.keys(payload).length) {
+            publish(payload);
+        }
+    }
+}
+
+module.exports = Receive;

lib/extension/responder.js

@@ -1,75 +0,0 @@
-const Zcl = require('zigbee-herdsman/dist/zcl');
-
-const OneJanuary2000 = new Date('January 01, 2000 00:00:00').getTime();
-
-/**
- * This extensions is reponsible for respondig to device requests.
- */
-class ExtensionResponder {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-
-        this.onZclFoundation = this.onZclFoundation.bind(this);
-        this.configured = [];
-    }
-
-    setupOnZclFoundation(device) {
-        if (device && device.epList && !this.configured.includes(device.ieeeAddr)) {
-            this.configured.push(device.ieeeAddr);
-
-            device.epList.forEach((epID) => {
-                const ep = this.zigbee.getEndpoint(device.ieeeAddr, epID);
-                ep.onZclFoundation = this.onZclFoundation;
-            });
-        }
-    }
-
-    onZigbeeStarted() {
-        this.zigbee.getAllClients().forEach((device) => this.setupOnZclFoundation(device));
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        this.setupOnZclFoundation(device);
-    }
-
-    readResponse(message, endpoint) {
-        const clusterID = message.clusterid;
-        const cluster = Zcl.getClusterLegacy(clusterID).key;
-        const attributes = message.zclMsg.payload.map((p) => Zcl.getAttributeLegacy(message.clusterid, p.attrId));
-        const response = [];
-
-        attributes.forEach((attribute) => {
-            if (cluster === 'genTime' && attribute.key === 'time') {
-                const time = Math.round(((new Date()).getTime() - OneJanuary2000) / 1000);
-                response.push(this.createReadResponseRec(clusterID, attribute.value, time));
-            }
-        });
-
-        this.zigbee.publish(
-            endpoint.device.ieeeAddr, 'device', cluster, 'readRsp', 'foundation', response,
-            {direction: 1, seqNum: message.zclMsg.seqNum, disDefaultRsp: 1}, endpoint.epId,
-        );
-    }
-
-    createReadResponseRec(cId, attrId, value) {
-        return {
-            attrId: attrId,
-            status: 0,
-            attrData: value,
-            dataType: Zcl.getAttributeTypeLegacy(cId, attrId).value,
-        };
-    }
-
-    onZclFoundation(message, endpoint) {
-        const cmd = message.zclMsg.cmdId;
-
-        if (cmd === 'read') {
-            this.readResponse(message, endpoint);
-        }
-    }
-}
-
-module.exports = ExtensionResponder;

lib/extension/softReset.js

@@ -1,63 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-
-/**
- * This extensions soft resets the ZNP after a certain timeout.
- */
-class SoftReset {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
-    }
-
-    onZigbeeStarted() {
-        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
-        this.resetTimer();
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    resetTimer() {
-        if (this.timeout === 0) {
-            return;
-        }
-
-        this.clearTimer();
-        this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
-    }
-
-    handleTimeout() {
-        logger.warn('Soft reset timeout triggered');
-
-        this.zigbee.softReset((error) => {
-            if (error) {
-                logger.warn('Soft reset failed, trying stop/start');
-                this.zigbee.stop((error) => {
-                    logger.warn('Zigbee stopped');
-                    this.zigbee.start((error) => {
-                        if (error) {
-                            logger.error('Failed to restart!');
-                        }
-                    });
-                });
-            } else {
-                logger.warn('Soft resetted ZNP due to timeout');
-            }
-
-            this.resetTimer();
-        });
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        this.resetTimer();
-    }
-}
-
-module.exports = SoftReset;

lib/extension/xiaomi.js

@@ -1,70 +0,0 @@
-const utils = require('../util/utils');
-const interval = utils.secondsToMilliseconds(60);
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-
-/**
- * This extensions handles Xiaomi devices.
- * - Marks Xiaomi devices as online.
- * - Polls Xiaomi routers to keep them awake.
- */
-class Xiaomi {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-    }
-
-    onZigbeeStarted() {
-        // Set all Xiaomi devices to be online, so shepherd won't try
-        // to query info from devices (which would fail because they go to sleep).
-        const devices = this.zigbee.getAllClients();
-        devices.forEach((d) => {
-            if (utils.isXiaomiDevice(d)) {
-                const device = this.zigbee.shepherd.find(d.ieeeAddr, 1);
-                if (device) {
-                    device.getDevice().update({
-                        status: 'online',
-                        joinTime: Math.floor(Date.now() / 1000),
-                    });
-                }
-            }
-        });
-
-        /**
-         * Device availability already pings Xiaomi devices.
-         */
-        if (!settings.get().advanced.availability_timeout) {
-            this.startTimer();
-        } else {
-            logger.debug('Disabled pinging of Xiaomi devices in Xiaomi extension because availability is enabled');
-        }
-    }
-
-    startTimer() {
-        this.clearTimer();
-        this.timer = setInterval(() => this.handleInterval(), interval);
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    stop() {
-        this.clearTimer();
-    }
-
-    ping(ieeeAddr) {
-        this.zigbee.ping(ieeeAddr, 'error', null, 'basic');
-    }
-
-    handleInterval() {
-        this.zigbee.getAllClients()
-            .filter((d) => utils.isXiaomiDevice(d) && utils.isRouter(d) && !utils.isBatteryPowered(d))
-            .forEach((d) => this.ping(d.ieeeAddr));
-    }
-}
-
-module.exports = Xiaomi;

lib/model/device.ts

@@ -0,0 +1,71 @@
+/* eslint-disable brace-style */
+import * as settings from '../util/settings';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+
+export default class Device {
+    private device: ZHDevice;
+    private _definition: Definition;
+
+    get endpoints(): ZHEndpoint[] {return this.device.endpoints;}
+    get zhDevice(): ZHDevice {return this.device;}
+    get ieeeAddr(): string {return this.device.ieeeAddr;}
+    get ID(): string {return this.device.ieeeAddr;}
+    get settings(): DeviceSettings {return {...settings.get().device_options, ...settings.getDevice(this.ieeeAddr)};}
+    get name(): string {
+        return this.type === 'Coordinator' ? 'Coordinator' : this.settings?.friendlyName || this.ieeeAddr;}
+    get lastSeen(): number {return this.device.lastSeen;}
+    get modelID(): string {return this.device.modelID;}
+    get softwareBuildID(): string {return this.device.softwareBuildID;}
+    get dateCode(): string {return this.device.dateCode;}
+    get interviewCompleted(): boolean {return this.device.interviewCompleted;}
+    get networkAddress(): number {return this.device.networkAddress;}
+    get manufacturerName(): string {return this.device.manufacturerName;}
+    get interviewing(): boolean {return this.device.interviewing;}
+    get type(): 'Coordinator' | 'Router' | 'EndDevice' | 'Unknown' | 'GreenPower' {return this.device.type;}
+    get powerSource(): string {return this.device.powerSource;}
+    get definition(): Definition | undefined {
+        if (!this._definition && !this.device.interviewing) {
+            this._definition = zhc.findByDevice(this.device);
+        }
+        return this._definition;
+    }
+
+    constructor(device: ZHDevice) {
+        this.device = device;
+    }
+
+    async ping(disableRecovery: boolean): Promise<void> {await this.device.ping(disableRecovery);}
+    async removeFromNetwork(): Promise<void> {await this.device.removeFromNetwork();}
+
+    endpoint(key?: string): ZHEndpoint {
+        let endpoint: ZHEndpoint;
+        if (key == null) key = 'default';
+
+        if (!isNaN(Number(key))) {
+            endpoint = this.device.getEndpoint(Number(key));
+        } else if (this.definition?.endpoint) {
+            const ID = this.definition?.endpoint?.(this.device)[key];
+            if (ID) endpoint = this.device.getEndpoint(ID);
+            else if (key === 'default') endpoint = this.device.endpoints[0];
+            else return null;
+        } else {
+            /* istanbul ignore next */
+            if (key !== 'default') return null;
+            endpoint = this.device.endpoints[0];
+        }
+
+        return endpoint;
+    }
+
+    isXiaomiDevice(): boolean {
+        const xiaomiManufacturerID = [4151, 4447];
+        /* istanbul ignore next */
+        return this.zhDevice.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(this.zhDevice.manufacturerID) &&
+            (!this.zhDevice.manufacturerName || !this.zhDevice.manufacturerName.startsWith('Trust'));
+    }
+
+    isRouter(): boolean {return this.zhDevice.type === 'Router';}
+    lqi(): Promise<LQI> {return this.zhDevice.lqi();}
+    routingTable(): Promise<RoutingTable> {return this.zhDevice.routingTable();}
+}

lib/model/group.ts

@@ -0,0 +1,26 @@
+/* eslint-disable brace-style */
+import * as settings from '../util/settings';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+
+export default class Group {
+    private group: ZHGroup;
+
+    get zhGroup(): ZHGroup {return this.group;}
+    get ID(): number {return this.group.groupID;}
+    get settings(): GroupSettings {return settings.getGroup(this.ID);}
+    get name(): string {return this.settings?.friendlyName || this.ID.toString();}
+    get members(): ZHEndpoint[] {return this.group.members;}
+
+    constructor(group: ZHGroup) {
+        this.group = group;
+    }
+
+    membersDefinitions(): Definition[] {
+        return this.members.map((m) => zhc.findByDevice(m.getDevice())).filter((d) => d) as Definition[];
+    }
+
+    membersIeeeAddr(): string[] {
+        return this.members.map((m) => m.getDevice().ieeeAddr);
+    }
+}

lib/mqtt.js

@@ -1,136 +0,0 @@
-const mqtt = require('mqtt');
-const logger = require('./util/logger');
-const settings = require('./util/settings');
-const fs = require('fs');
-
-class MQTT {
-    constructor() {
-        this.onConnect = this.onConnect.bind(this);
-        this.onMessage = this.onMessage.bind(this);
-        this.messageHandler = null;
-    }
-
-    connect(messageHandler, callback) {
-        const mqttSettings = settings.get().mqtt;
-        logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
-
-        const options = {
-            will: {
-                topic: `${settings.get().mqtt.base_topic}/bridge/state`,
-                payload: 'offline',
-                retain: true,
-            },
-        };
-
-        if (mqttSettings.ca) {
-            logger.debug(`MQTT SSL/TLS: Path to CA certificate = ${mqttSettings.ca}`);
-            const ca = fs.readFileSync(mqttSettings.ca);
-
-            if (ca) {
-                options.ca = ca;
-            } else {
-                logger.error(`Error loading CA certificate for MQTT SSL/TLS configuration.`);
-            }
-        }
-
-        if (mqttSettings.key && mqttSettings.cert) {
-            logger.debug(`MQTT SSL/TLS: Path to client key = ${mqttSettings.key}`);
-            logger.debug(`MQTT SSL/TLS: Path to client certificate = ${mqttSettings.cert}`);
-
-            const key = fs.readFileSync(mqttSettings.key);
-            const cert = fs.readFileSync(mqttSettings.cert);
-
-            if (key && cert) {
-                options.key = key;
-                options.cert = cert;
-            } else {
-                logger.error(`Error loading key and/or certificate for MQTT SSL/TLS client authentication.`);
-            }
-        }
-
-        if (mqttSettings.user && mqttSettings.password) {
-            options.username = mqttSettings.user;
-            options.password = mqttSettings.password;
-        }
-
-        if (mqttSettings.client_id) {
-            logger.debug(`Using MQTT client ID: '${mqttSettings.client_id}'`);
-            options.clientId = mqttSettings.client_id;
-        }
-
-        if (mqttSettings.hasOwnProperty('reject_unauthorized') && !mqttSettings.reject_unauthorized) {
-            logger.debug(`MQTT reject_unauthorized set false, ignoring certificate warnings.`);
-            options.rejectUnauthorized = false;
-        }
-
-        this.client = mqtt.connect(mqttSettings.server, options);
-
-        // Register callbacks.
-        this.client.on('connect', () => {
-            this.onConnect();
-            callback();
-        });
-
-        this.client.on('message', this.onMessage);
-
-        // Set timer at interval to check if connected to MQTT server.
-        const interval = 10 * 1000; // seconds * 1000.
-        this.connectionTimer = setInterval(() => {
-            if (this.client.reconnecting) {
-                logger.error('Not connected to MQTT server!');
-            }
-        }, interval);
-
-        this.messageHandler = messageHandler;
-    }
-
-    disconnect() {
-        clearTimeout(this.connectionTimer);
-        this.connectionTimer = null;
-
-        this.publish('bridge/state', 'offline', {retain: true, qos: 0}, () => {
-            logger.info('Disconnecting from MQTT server');
-            this.client.end();
-        });
-    }
-
-    onConnect() {
-        logger.info('Connected to MQTT server');
-        this.publish('bridge/state', 'online', {retain: true, qos: 0});
-    }
-
-    subscribe(topic) {
-        this.client.subscribe(topic);
-    }
-
-    onMessage(topic, message) {
-        if (this.messageHandler) {
-            this.messageHandler(topic, message);
-        }
-    }
-
-    publish(topic, payload, options, callback, base=settings.get().mqtt.base_topic) {
-        topic = `${base}/${topic}`;
-
-        if (!this.client || this.client.reconnecting) {
-            logger.error(`Not connected to MQTT server!`);
-            logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
-            return;
-        }
-
-        logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
-        this.client.publish(topic, payload, options, callback);
-    }
-
-    log(type, message, meta=null) {
-        const payload = {type, message};
-
-        if (meta) {
-            payload.meta = meta;
-        }
-
-        this.publish('bridge/log', JSON.stringify(payload), {retain: false});
-    }
-}
-
-module.exports = MQTT;

lib/mqtt.ts

@@ -0,0 +1,153 @@
+import mqtt from 'mqtt';
+import logger from './util/logger';
+import * as settings from './util/settings';
+import {seconds} from './util/utils';
+import fs from 'fs';
+
+export default class MQTT {
+    private publishedTopics: Set<string> = new Set();
+    private connectionTimer: NodeJS.Timeout;
+    private client: mqtt.MqttClient;
+    private eventBus: EventBus;
+
+    constructor(eventBus: EventBus) {
+        this.eventBus = eventBus;
+        this.onMessage = this.onMessage.bind(this);
+        this.onConnect = this.onConnect.bind(this);
+    }
+
+    async connect(): Promise<void> {
+        const mqttSettings = settings.get().mqtt;
+        logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
+
+        const options: mqtt.IClientOptions = {
+            will: {
+                topic: `${settings.get().mqtt.base_topic}/bridge/state`,
+                payload: 'offline',
+                retain: settings.get().mqtt.force_disable_retain ? false : true,
+                qos: 1,
+            },
+        };
+
+        if (mqttSettings.version) {
+            options.protocolVersion = mqttSettings.version;
+        }
+
+        if (mqttSettings.keepalive) {
+            logger.debug(`Using MQTT keepalive: ${mqttSettings.keepalive}`);
+            options.keepalive = mqttSettings.keepalive;
+        }
+
+        if (mqttSettings.ca) {
+            logger.debug(`MQTT SSL/TLS: Path to CA certificate = ${mqttSettings.ca}`);
+            options.ca = fs.readFileSync(mqttSettings.ca);
+        }
+
+        if (mqttSettings.key && mqttSettings.cert) {
+            logger.debug(`MQTT SSL/TLS: Path to client key = ${mqttSettings.key}`);
+            logger.debug(`MQTT SSL/TLS: Path to client certificate = ${mqttSettings.cert}`);
+            options.key = fs.readFileSync(mqttSettings.key);
+            options.cert = fs.readFileSync(mqttSettings.cert);
+        }
+
+        if (mqttSettings.user && mqttSettings.password) {
+            logger.debug(`Using MQTT login with username: ${mqttSettings.user}`);
+            options.username = mqttSettings.user;
+            options.password = mqttSettings.password;
+        } else {
+            logger.debug(`Using MQTT anonymous login`);
+        }
+
+        if (mqttSettings.client_id) {
+            logger.debug(`Using MQTT client ID: '${mqttSettings.client_id}'`);
+            options.clientId = mqttSettings.client_id;
+        }
+
+        if (mqttSettings.hasOwnProperty('reject_unauthorized') && !mqttSettings.reject_unauthorized) {
+            logger.debug(`MQTT reject_unauthorized set false, ignoring certificate warnings.`);
+            options.rejectUnauthorized = false;
+        }
+
+        return new Promise((resolve) => {
+            this.client = mqtt.connect(mqttSettings.server, options);
+
+            const onConnect = this.onConnect;
+            this.client.on('connect', async () => {
+                await onConnect();
+                resolve();
+            });
+
+            this.client.on('message', this.onMessage);
+        });
+    }
+
+    private async onConnect(): Promise<void> {
+        // Set timer at interval to check if connected to MQTT server.
+        clearTimeout(this.connectionTimer);
+        this.connectionTimer = setInterval(() => {
+            if (this.client.reconnecting) {
+                logger.error('Not connected to MQTT server!');
+            }
+        }, seconds(10));
+
+        logger.info('Connected to MQTT server');
+        this.subscribe(`${settings.get().mqtt.base_topic}/#`);
+        await this.publish('bridge/state', 'online', {retain: true, qos: 0});
+    }
+
+    async disconnect(): Promise<void> {
+        clearTimeout(this.connectionTimer);
+        await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
+        logger.info('Disconnecting from MQTT server');
+        this.client.end();
+    }
+
+    subscribe(topic: string): void {
+        this.client.subscribe(topic);
+    }
+
+    public onMessage(topic: string, message: string): void {
+        // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
+        if (!this.publishedTopics.has(topic)) {
+            this.eventBus.emitMQTTMessage({topic, message: message + ''});
+        }
+    }
+
+    isConnected(): boolean {
+        return this.client && !this.client.reconnecting;
+    }
+
+    async publish(topic: string, payload: string, options: {qos?: mqtt.QoS, retain?: boolean}={},
+        base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true,
+    ): Promise<void> {
+        const defaultOptions: {qos: mqtt.QoS, retain: boolean} = {qos: 0, retain: false};
+        topic = `${base}/${topic}`;
+
+        if (skipReceive) {
+            this.publishedTopics.add(topic);
+        }
+
+        this.eventBus.emitMQTTMessagePublished({topic, payload, options: {...defaultOptions, ...options}});
+
+        if (!this.isConnected()) {
+            if (!skipLog) {
+                logger.error(`Not connected to MQTT server!`);
+                logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
+            }
+            return;
+        }
+
+        if (!skipLog) {
+            logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
+        }
+
+        const actualOptions: mqtt.IClientPublishOptions = {...defaultOptions, ...options};
+        if (settings.get().mqtt.force_disable_retain) {
+            actualOptions.retain = false;
+        }
+
+        return new Promise((resolve) => {
+            this.client.publish(topic, payload, actualOptions, () => resolve());
+        });
+    }
+}

lib/state.js

@@ -1,26 +1,23 @@
 const logger = require('./util/logger');
 const data = require('./util/data');
-const fs = require('fs');
 const settings = require('./util/settings');
+const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'action_color_temperature', 'action_color',
-    'action_group', 'group_list', 'group_capacity', 'no_occupancy_since',
-    'step_mode', 'transition_time', 'duration', 'elapsed',
+    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
+    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
+    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
 ];
 
 class State {
-    constructor() {
+    constructor(eventBus) {
         this.state = {};
         this.file = data.joinPath('state.json');
         this.timer = null;
-        this.stateChangeListeners = [];
-
-        this.handleSettingsChanged = this.handleSettingsChanged.bind(this);
+        this.eventBus = eventBus;
     }
 
     start() {
@@ -29,31 +26,6 @@ class State {
         // Save the state on every interval
         this.clearTimer();
         this.timer = setInterval(() => this.save(), saveInterval);
-
-        // Listen for on settings changed events.
-        settings.addOnChangeHandler(this.handleSettingsChanged);
-
-        this.checkLastSeen();
-    }
-
-    handleSettingsChanged() {
-        this.checkLastSeen();
-    }
-
-    registerOnStateChangeListener(listener) {
-        this.stateChangeListeners.push(listener);
-    }
-
-    checkLastSeen() {
-        if (settings.get().advanced.last_seen === 'disable') {
-            Object.values(this.state).forEach((s) => {
-                if (s.hasOwnProperty('last_seen')) {
-                    delete s.last_seen;
-                }
-            });
-
-            this.save();
-        }
     }
 
     clearTimer() {
@@ -82,36 +54,64 @@ class State {
     }
 
     save() {
-        logger.debug(`Saving state to file ${this.file}`);
-        const json = JSON.stringify(this.state, null, 4);
-        fs.writeFileSync(this.file, json, 'utf8');
+        if (settings.get().advanced.cache_state_persistent) {
+            logger.debug(`Saving state to file ${this.file}`);
+            const json = JSON.stringify(this.state, null, 4);
+            try {
+                fs.writeFileSync(this.file, json, 'utf8');
+            } catch (e) {
+                logger.error(`Failed to write state to '${this.file}' (${e.message})`);
+            }
+        } else {
+            logger.debug(`Not saving state`);
+        }
     }
 
-    exists(ieeeAddr) {
-        return this.state.hasOwnProperty(ieeeAddr);
+    exists(ID) {
+        return this.state.hasOwnProperty(ID);
     }
 
-    get(ieeeAddr) {
-        return this.state[ieeeAddr];
+    get(ID) {
+        return this.state[ID];
     }
 
-    set(ieeeAddr, state) {
-        const toState = objectAssignDeep.noMutate(state);
-        dontCacheProperties.forEach((property) => {
-            if (toState.hasOwnProperty(property)) {
+    set(ID, update, reason=null) {
+        const fromState = this.state[ID] || {};
+        const toState = objectAssignDeep.noMutate(fromState, update);
+        const result = {...toState};
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
                 delete toState[property];
             }
-        });
+        }
 
-        const fromState = this.state[ieeeAddr];
-        this.stateChangeListeners.forEach((listener) => listener(ieeeAddr, fromState, toState));
-
-        this.state[ieeeAddr] = toState;
+        this.state[ID] = toState;
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason, update});
+        return result;
     }
 
-    remove(ieeeAddr) {
-        if (this.exists(ieeeAddr)) {
-            delete this.state[ieeeAddr];
+    removeKey(ID, path) {
+        if (this.exists(ID)) {
+            let state = this.state[ID];
+            for (let i = 0; i < path.length; i++) {
+                const key = path[i];
+                if (i === path.length - 1) {
+                    delete state[key];
+                } else {
+                    if (state[key]) {
+                        state = state[key];
+                    } else {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    remove(ID) {
+        if (this.exists(ID)) {
+            delete this.state[ID];
         }
     }
 }

lib/types/types.ts

@@ -0,0 +1,279 @@
+import type {
+    Device as ZZHDevice,
+    Group as ZZHGroup,
+    Endpoint as ZZHEndpoint,
+} from 'zigbee-herdsman/dist/controller/model';
+
+import {
+    NetworkParameters as ZHNetworkParameters,
+    CoordinatorVersion as ZHCoordinatorVersion,
+    LQI as ZHLQI,
+    RoutingTable as ZHRoutingTable,
+    RoutingTableEntry as ZHRoutingTableEntry,
+} from 'zigbee-herdsman/dist/adapter/tstype';
+
+import * as D from 'lib/model/device';
+import * as Z from 'lib/zigbee';
+import * as E from 'lib/eventBus';
+import * as M from 'lib/mqtt';
+// TODO: check all
+
+declare global {
+    type RecursivePartial<T> = {
+        [P in keyof T]?: RecursivePartial<T[P]>;
+    };
+
+    type EventBus = E.default;
+
+    type MQTT = M.default;
+
+    type Zigbee = Z.default;
+
+    type Device = D.default;
+
+    type ZHEndpoint = ZZHEndpoint;
+
+    type ZHDevice = ZZHDevice;
+    type LQI = ZHLQI;
+    type RoutingTable = ZHRoutingTable;
+    type RoutingTableEntry = ZHRoutingTableEntry;
+
+    type ZHGroup = ZZHGroup;
+
+    type CoordinatorVersion = ZHCoordinatorVersion;
+
+    type NetworkParameters = ZHNetworkParameters;
+
+    type ZigbeeEventType = 'deviceLeave' | 'deviceAnnounce';
+
+    interface ZigbeeEventData {
+        ieeeAddr: string;
+    }
+
+    /* eslint-disable */
+    // Controller
+    interface KeyValue {
+        [s: string]: any,
+    }
+
+    interface Settings {
+        homeassistant?: boolean,
+        devices?: {[s: string]: {friendly_name: string, retention?: number}},
+        groups?: {[s: string]: {friendly_name: string, devices?: string[]}},
+        passlist: string[],
+        blocklist: string[],
+        whitelist: string[],
+        ban: string[],
+        availability?: boolean | {
+            active?: {timeout?: number},
+            passive?: {timeout?: number}
+        },
+        permit_join: boolean,
+        frontend?: {
+            auth_token?: string,
+            host?: string,
+            port?: number,
+        },
+        mqtt: {
+            base_topic: string,
+            include_device_information: boolean,
+            force_disable_retain: boolean
+            version?: number,
+            user?: string,
+            password?: string,
+            server: string,
+            ca?: string,
+            keepalive?: number,
+            key?: string,
+            cert?: string,
+            client_id?: string,
+            reject_unauthorized?: boolean,
+        },
+        serial: {
+            disable_led?: boolean,
+            port?: string,
+            adapter?: 'deconz' | 'zstack' | 'ezsp' | 'zigate'
+        },
+        device_options: {[s: string]: unknown},
+        map_options: {
+            graphviz: {
+                colors: {
+                    fill: {
+                        enddevice: string,
+                        coordinator: string,
+                        router: string,
+                    },
+                    font: {
+                        coordinator: string,
+                        router: string,
+                        enddevice: string,
+                    },
+                    line: {
+                        active: string,
+                        inactive: string,
+                    },
+                },
+            },
+        },
+        experimental: {
+            output: 'json' | 'attribute' | 'attribute_and_json',
+            availability_new?: boolean,
+            transmit_power?: number,
+        },
+        advanced: {
+            legacy_api: boolean,
+            log_rotation: boolean,
+            log_symlink_current: boolean,
+            log_output: ('console' | 'file')[],
+            log_directory: string,
+            log_file: string,
+            log_level: 'debug' | 'info' | 'error' | 'warn',
+            log_syslog: {},
+            soft_reset_timeout: number,
+            pan_id: number | 'GENERATE',
+            ext_pan_id: number[],
+            channel: number,
+            adapter_concurrent: number | null,
+            adapter_delay: number | null,
+            availability_timeout: number,
+            availability_blocklist: string[],
+            availability_passlist: string[],
+            availability_blacklist: string[],
+            availability_whitelist: string[],
+            cache_state: boolean,
+            cache_state_persistent: boolean,
+            cache_state_send_on_startup: boolean,
+            last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' |  'epoch',
+            elapsed: boolean,
+            network_key: number[] | 'GENERATE',
+            report: boolean,
+            homeassistant_discovery_topic: string,
+            homeassistant_status_topic: string,
+            homeassistant_legacy_entity_attributes: boolean,
+            homeassistant_legacy_triggers: boolean,
+            timestamp_format: string,
+            baudrate?: number,
+            rtscts?: boolean,
+            ikea_ota_use_test_url?: boolean,
+        },
+        ota: {
+            update_check_interval: number,
+            disable_automatic_update_check: boolean
+        },
+        external_converters: string[],
+    }
+
+    interface DeviceSettings {
+        friendlyName: string,
+        ID: string,
+        retention?: number,
+        availability?: boolean | {timeout: number},
+        optimistic?: boolean,
+        retrieve_state?: boolean,
+        debounce?: number,
+        debounce_ignore?: string[],
+        filtered_optimistic?: string[],
+        icon?: string,
+        homeassistant?: KeyValue,
+        legacy?: boolean,
+    }
+
+    interface GroupSettings {
+        friendlyName: string,
+        devices: string[],
+        ID: number,
+        optimistic?: boolean,
+        filtered_optimistic?: string[],
+        retrieve_state?: boolean,
+        homeassistant?: KeyValue,
+    }
+
+    type EntitySettings = {
+        type: 'device' | 'group'
+        ID: number | string,
+        friendlyName: string,
+    }
+
+    interface ResolvedEntity {
+        type: 'device' | 'group',
+        device: ZHDevice,
+        name: string,
+    }
+
+    interface ToZigbeeConverterGetMeta {message: {}, mapped: Definition | Definition[]}
+
+    interface ToZigbeeConverterResult {state: KeyValue, membersState: {[s: string]: KeyValue}, readAfterWriteTime?: number}
+
+    interface ToZigbeeConverter {
+        key: string[],
+        convertGet?: (entity: ZHEndpoint | ZHGroup, key: string, meta: ToZigbeeConverterGetMeta) => Promise<void>
+        convertSet?: (entity: ZHEndpoint | ZHGroup, key: string, value: any, meta: {state: KeyValue}) => Promise<ToZigbeeConverterResult>
+    }
+
+    // interface Logger {
+    //     error: (message: string) => void;
+    //     warn: (message: string) => void;
+    //     debug: (message: string) => void;
+    //     info: (message: string) => void;
+    // }
+
+    interface FromZigbeeConverter {
+        cluster: string,
+        type: string[] | string,
+        convert: (model: Definition, msg: KeyValue, publish: (payload: KeyValue) => void, options: KeyValue,
+            meta: {state: KeyValue, logger: any, device: ZHDevice}) => KeyValue,
+    }
+
+    interface DefinitionExposeFeature {name: string, endpoint?: string, property: string, value_max?: number, value_min?: number, value_off?: string, value_on?: string, value_step?: number, values: string[], access: number}
+
+    interface DefinitionExpose {type: string, name?: string, features?: DefinitionExposeFeature[], endpoint?: string, values?: string[], value_off?: string, value_on?: string, access: number, property: string, unit?: string, value_min?: number, value_max?: number}
+
+    interface Definition  {
+        model: string
+        endpoint?: (device: ZHDevice) => {[s: string]: number}
+        toZigbee: ToZigbeeConverter[]
+        fromZigbee: FromZigbeeConverter[]
+        icon?: string
+        description: string
+        vendor: string
+        exposes: DefinitionExpose[] // TODO
+        configure?: (device: ZHDevice, coordinatorEndpoint: ZZHEndpoint, logger: unknown) => Promise<void>;
+        onEvent?: (type: string, data: KeyValue, device: ZHDevice, settings: KeyValue) => Promise<void>;
+        ota?: {
+            isUpdateAvailable: (device: ZHDevice, logger: unknown, data?: KeyValue) => Promise<boolean>;
+            updateToLatest: (device: ZHDevice, logger: unknown, onProgress: (progress: number, remaining: number) => void) => Promise<void>;
+        }
+    }
+
+    interface ResolvedDevice {
+        type: 'device',
+        definition?: Definition,
+        name: string,
+        endpoint: ZHEndpoint,
+        device: ZHDevice,
+        settings: {
+            friendlyName: string,
+            availability?: {timeout?: number} | boolean,
+        }
+    }
+
+    interface TempMQTT {
+        publish: (topic: string, payload: string, options: {}, base?: string, skipLog?: boolean, skipReceive?: boolean) => Promise<void>;
+    }
+
+    interface TempState {
+        get: (ID: string | number) => KeyValue | null;
+        remove: (ID: string | number) => void;
+        removeKey: (ID: string, keys: string[]) => void;
+        exists: (ID: string) => boolean;
+    }
+
+    interface ExternalConverterClass {
+        new(zigbee: Zigbee, mqtt: MQTT, state: TempState, publishEntityState: PublishEntityState,
+            eventBus: EventBus, settings: unknown, logger: unknown): ExternalConverterClass;
+    }
+
+    interface MQTTResponse {data: KeyValue, status: string, error?: string, transaction?: string}
+
+    type PublishEntityState = (ID: string | number, payload: KeyValue, stateChangeReason?: 'publishDebounce') => void;
+}

lib/util/data.js

@@ -1,33 +0,0 @@
-const path = require('path');
-const fs = require('fs');
-
-let dataPath = null;
-
-function load() {
-    if (process.env.ZIGBEE2MQTT_DATA) {
-        dataPath = process.env.ZIGBEE2MQTT_DATA;
-    } else {
-        dataPath = path.join(__dirname, '..', '..', 'data');
-        dataPath = path.normalize(dataPath);
-    }
-}
-
-load();
-
-function joinPathStorage(file) {
-    const storagePath = path.join(dataPath, '.storage');
-    if (!fs.existsSync(storagePath)) {
-        fs.mkdirSync(storagePath);
-    }
-
-    return path.join(storagePath, file);
-}
-
-module.exports = {
-    joinPath: (file) => path.join(dataPath, file),
-    joinPathStorage: (file) => joinPathStorage(file),
-    getPath: () => dataPath,
-
-    // For test only.
-    _reload: () => load(),
-};

lib/util/data.ts

@@ -0,0 +1,27 @@
+import path from 'path';
+
+let dataPath: string = null;
+
+function load(): void {
+    if (process.env.ZIGBEE2MQTT_DATA) {
+        dataPath = process.env.ZIGBEE2MQTT_DATA;
+    } else {
+        dataPath = path.join(__dirname, '..', '..', 'data');
+        dataPath = path.normalize(dataPath);
+    }
+}
+
+load();
+
+export function joinPath(file: string): string {
+    return path.join(dataPath, file);
+}
+
+export function getPath(): string {
+    return dataPath;
+}
+
+// eslint-disable-next-line camelcase
+export function __testingOnly_reload(): void {
+    load();
+}

lib/util/fs.js

@@ -1,16 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-
-function readYaml(file) {
-    return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-}
-
-function readYamlIfExists(file) {
-    return fs.existsSync(file) ? readYaml(file) : null;
-}
-
-function writeYaml(file, content) {
-    fs.writeFileSync(file, yaml.safeDump(content));
-}
-
-module.exports = {readYaml, readYamlIfExists, writeYaml};

lib/util/logger.js

@@ -5,61 +5,109 @@ const path = require('path');
 const fs = require('fs');
 const fx = require('mkdir-recursive');
 const rimraf = require('rimraf');
+const colorizer = winston.format.colorize();
+const assert = require('assert');
 
-// Determine the log level.
-const level = settings.get().advanced.log_level;
+// What transports to enable
+const output = settings.get().advanced.log_output;
 
-// Directoy to log to
+// Directory to log to
 const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
 const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
 
-// Make sure that log directoy exsists
-fx.mkdirSync(directory);
+// Make sure that log directoy exsists when not logging to stdout only
+if (output.includes('file')) {
+    fx.mkdirSync(directory);
 
-// Custom level
-const levels = {
-    levels: {
-        error: 0,
-        warn: 1,
-        info: 2,
-        debug: 3,
-        trace: 4,
-    },
-    colors: {
-        error: 'red',
-        warn: 'yellow',
-        info: 'green',
-        debug: 'blue',
-        trace: 'magenta',
-    },
+    if (settings.get().advanced.log_symlink_current) {
+        const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
+        const actual = './' + timestamp;
+        if (fs.existsSync(current)) {
+            fs.unlinkSync(current);
+        }
+        fs.symlinkSync(actual, current);
+    }
+}
+
+const z2mToWinstonLevel = (level) => level === 'warn' ? 'warning' : level;
+const winstonToZ2mLevel = (level) => level === 'warning' ? 'warn' : level;
+
+// Determine the log level.
+let level = settings.get().advanced.log_level;
+const validLevels = ['info', 'error', 'warn', 'debug'];
+assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
+level = z2mToWinstonLevel(level);
+
+const levelWithCompensatedLength = {
+    'info': 'info ',
+    'error': 'error',
+    'warn': 'warn ',
+    'debug': 'debug',
 };
 
+/* istanbul ignore next */
+const timestampFormat = () => moment().format(settings.get().advanced.timestamp_format);
+
+// Setup default console logger
+const transportsToUse = [
+    new winston.transports.Console({
+        level,
+        silent: !output.includes('console'),
+        format: winston.format.combine(
+            winston.format.timestamp({format: timestampFormat}),
+            winston.format.printf(/* istanbul ignore next */(info) => {
+                let {timestamp, level, message} = info;
+                level = winstonToZ2mLevel(level);
+                const prefix = colorizer.colorize(level, `Zigbee2MQTT:${levelWithCompensatedLength[level]}`);
+                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+            }),
+        ),
+    }),
+];
+
+// Add file logger when enabled
+// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            let {timestamp, level, message} = info;
+            level = winstonToZ2mLevel(level);
+            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
+if (output.includes('file')) {
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
+}
+
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    require('winston-syslog').Syslog;
+    const options = {
+        app_name: 'Zigbee2MQTT',
+        format: winston.format.printf(/* istanbul ignore next */(info) => {
+            return `${info.message}`;
+        }),
+        ...settings.get().advanced.log_syslog,
+    };
+    if (options.hasOwnProperty('type')) options.type = options.type.toString();
+    transportsToUse.push(new winston.transports.Syslog(options));
+}
+
 // Create logger
-const logger = new winston.Logger({
-    levels: levels.levels,
-    transports: [
-        new winston.transports.File({
-            filename: path.join(directory, 'log.txt'),
-            json: false,
-            level,
-            maxFiles: 3, // Keep last 3 files
-            maxsize: 10000000, // 10MB
-            timestamp: () => new Date().toLocaleString(),
-        }),
-        new winston.transports.Console({
-            timestamp: () => new Date().toLocaleString(),
-            formatter: (options) =>
-                winston.config.colorize(options.level, '  zigbee2mqtt:' + options.level.toLowerCase()) + ' ' +
-                options.timestamp() + ' ' + (options.message ? options.message : '') +
-                (options.meta && Object.keys(options.meta).length ? '\n\t'+ JSON.stringify(options.meta) : '' ),
-        }),
-    ],
-});
-
-// Add colors
-winston.addColors(levels.colors);
-
-logger.transports.console.level = level;
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
 
 // Cleanup any old log directory.
 function cleanup() {
@@ -68,7 +116,7 @@ function cleanup() {
 
         let directories = fs.readdirSync(rootDirectory).map((d) => {
             d = path.join(rootDirectory, d);
-            return {path: d, birth: fs.statSync(d).birthtimeMs};
+            return {path: d, birth: fs.statSync(d).mtime};
         });
 
         directories.sort((a, b) => b.birth - a.birth);
@@ -80,7 +128,34 @@ function cleanup() {
     }
 }
 
+// Print to user what logging is enabled
+function logOutput() {
+    if (output.includes('file')) {
+        if (output.includes('console')) {
+            logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+        } else {
+            logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+        }
+        logger.cleanup();
+    } else if (output.includes('console')) {
+        logger.info(`Logging to console only'`);
+    }
+}
+
+logger.addTransport = (transport) => {
+    transport.level = transportsToUse[0].level;
+    logger.add(transport);
+};
 logger.cleanup = cleanup;
-logger.directory = directory;
+logger.logOutput = logOutput;
+logger.getLevel = () => winstonToZ2mLevel(transportsToUse[0].level);
+logger.setLevel = (level) => {
+    level = z2mToWinstonLevel(level);
+    logger.transports.forEach((transport) => transport.level = level);
+};
+
+// winston.config.syslog.levels doesnt have warn, but is required for syslog.
+/* istanbul ignore next */
+logger.warn = (message) => logger.warning(message);
 
 module.exports = logger;

lib/util/settings.js

@@ -1,420 +0,0 @@
-const data = require('./data');
-const file = data.joinPath('configuration.yaml');
-const objectAssignDeep = require(`object-assign-deep`);
-const path = require('path');
-const fs = require('./fs');
-const onChangeHandlers = [];
-
-const defaults = {
-    whitelist: [],
-    ban: [],
-    permit_join: false,
-    mqtt: {
-        include_device_information: false,
-    },
-    groups: {},
-    device_options: {},
-    map_options: {
-        graphviz: {
-            colors: {
-                fill: {
-                    enddevice: '#fff8ce',
-                    coordinator: '#e04e5d',
-                    router: '#4ea3e0',
-                },
-                font: {
-                    coordinator: '#ffffff',
-                    router: '#ffffff',
-                    enddevice: '#000000',
-                },
-                line: {
-                    active: '#009900',
-                    inactive: '#994444',
-                },
-            },
-        },
-    },
-    queue: {
-        delay: 250,
-        simultaneously: 5,
-    },
-    experimental: {
-        livolo: false,
-        // json or attribute
-        output: 'json',
-    },
-    advanced: {
-        log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
-        log_level: process.env.DEBUG ? 'debug' : 'info',
-        soft_reset_timeout: 0,
-        pan_id: 0x1a62,
-        ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
-        channel: 11,
-        baudrate: 115200,
-        rtscts: true,
-
-        // Availability timeout in seconds, disabled by default.
-        availability_timeout: 0,
-        availability_blacklist: [],
-
-        /**
-         * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
-         * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
-         *
-         * Therefore zigbee2mqtt BY DEFAULT caches all values and resend it with every message.
-         * advanced.cache_state in configuration.yaml allows to configure this.
-         * https://www.zigbee2mqtt.io/configuration/configuration.html
-         */
-        cache_state: true,
-
-        /**
-         * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
-         * "ISO_8601": ISO 8601 format
-         * "ISO_8601_local": Local ISO 8601 format (instead of UTC-based)
-         * "epoch": milliseconds elapsed since the UNIX epoch
-         * "disable": no last_seen attribute (default)
-         */
-        last_seen: 'disable',
-
-        // Optional: Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg
-        elapsed: false,
-
-        /**
-         * https://github.com/Koenkk/zigbee2mqtt/issues/685#issuecomment-449112250
-         *
-         * Network key will serve as the encryption key of your network.
-         * Changing this will require you to repair your devices.
-         */
-        network_key: [1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13],
-
-        /**
-         * Enables reporting feature
-         */
-        report: false,
-
-        /**
-         * Home Assistant discovery topic
-         */
-        homeassistant_discovery_topic: 'homeassistant',
-
-        /**
-         * Home Assistant status topic
-         */
-        homeassistant_status_topic: 'hass/status',
-    },
-};
-
-let _settings;
-
-function getSettings() {
-    if (!_settings) {
-        _settings = read();
-    }
-
-    if (_settings.hasOwnProperty('advanced') && !_settings.advanced) {
-        delete _settings.advanced;
-    }
-
-    return _settings;
-}
-
-function get() {
-    const _settings = objectAssignDeep.noMutate(defaults, getSettings());
-
-    if (!_settings.whitelist) {
-        _settings.whitelist = [];
-    }
-
-    return _settings;
-}
-
-function writeRead() {
-    write();
-    _settings = read();
-    onChangeHandlers.forEach((handler) => handler());
-}
-
-function write() {
-    const settings = getSettings();
-    const toWrite = objectAssignDeep.noMutate(settings);
-
-    // Read settings to check if we have to split devices/groups into separate file.
-    const actual = fs.readYaml(file);
-    if (typeof actual.devices === 'string') {
-        fs.writeYaml(data.joinPath(actual.devices), settings.devices);
-        toWrite.devices = actual.devices;
-    }
-
-    if (typeof actual.groups === 'string') {
-        fs.writeYaml(data.joinPath(actual.groups), settings.groups);
-        toWrite.groups = actual.groups;
-    }
-
-    fs.writeYaml(file, toWrite);
-}
-
-function read() {
-    const s = fs.readYaml(file);
-
-    // Read devices/groups configuration from separate file.
-    if (typeof s.devices === 'string') {
-        const file = data.joinPath(s.devices);
-        s.devices = fs.readYamlIfExists(file);
-    }
-
-    if (typeof s.groups === 'string') {
-        const file = data.joinPath(s.groups);
-        s.groups = fs.readYamlIfExists(file);
-    }
-
-    return s;
-}
-
-function set(path, value) {
-    let obj = getSettings();
-
-    for (let i = 0; i < path.length; i++) {
-        const key = path[i];
-        if (i === path.length - 1) {
-            obj[key] = value;
-        } else {
-            if (!obj[key]) {
-                obj[key] = {};
-            }
-
-            obj = obj[key];
-        }
-    }
-
-    writeRead();
-}
-
-const getDevices = () => getSettings().devices || [];
-
-const getDevice = (ieeeAddr) => getDevices()[ieeeAddr];
-
-const getGroups = () => getSettings().groups || {};
-
-const getGroup = (ID) => {
-    const group = getGroups()[ID];
-
-    if (group && !group.hasOwnProperty('devices')) {
-        group.devices = [];
-    }
-
-    return group;
-};
-
-
-function addDevice(ieeeAddr) {
-    const settings = getSettings();
-    if (!settings.devices) {
-        settings.devices = {};
-    }
-
-    settings.devices[ieeeAddr] = {friendly_name: ieeeAddr, retain: false};
-    writeRead();
-}
-
-function whitelistDevice(ieeeAddr) {
-    const settings = getSettings();
-    if (!settings.whitelist) {
-        settings.whitelist = [];
-    }
-
-    settings.whitelist.push(ieeeAddr);
-
-    writeRead();
-}
-
-function banDevice(ieeeAddr) {
-    const settings = getSettings();
-    if (!settings.ban) {
-        settings.ban = [];
-    }
-
-    settings.ban.push(ieeeAddr);
-
-    writeRead();
-}
-
-function removeDevice(ieeeAddr) {
-    const settings = getSettings();
-    if (!settings.devices || !settings.devices[ieeeAddr]) return;
-
-    delete settings.devices[ieeeAddr];
-    writeRead();
-}
-
-function addGroup(groupName) {
-    const settings = getSettings();
-    if (!settings.groups) {
-        settings.groups = {};
-    }
-
-    let ID = '1';
-    while (settings.groups.hasOwnProperty(ID)) {
-        ID = (Number.parseInt(ID) + 1).toString();
-    }
-
-    if (Object.values(settings.groups).find((g) => g.friendly_name === groupName)) {
-        return false; // Duplicate group names are not allowed
-    }
-
-    settings.groups[ID] = {friendly_name: groupName};
-    writeRead();
-
-    return true;
-}
-
-function addDeviceToGroup(ID, ieeeAddr) {
-    const settings = getSettings();
-    const group = settings.groups[ID];
-    if (!group.devices) {
-        group.devices = [];
-    }
-
-    if (!group.devices.includes(ieeeAddr)) {
-        group.devices.push(ieeeAddr);
-        writeRead();
-        return true;
-    } else {
-        return false;
-    }
-}
-
-function removeDeviceFromGroup(ID, ieeeAddr) {
-    const settings = getSettings();
-    const group = settings.groups[ID];
-
-    if (group.devices && group.devices.includes(ieeeAddr)) {
-        group.devices = group.devices.filter((d) => d != ieeeAddr);
-        writeRead();
-        return true;
-    } else {
-        return false;
-    }
-}
-
-function removeGroup(name) {
-    const settings = getSettings();
-    if (!settings.groups) return;
-
-    const ID = Object.keys(settings.groups).find((key) => {
-        return settings.groups[key].friendly_name === name;
-    });
-
-    if (ID) {
-        delete settings.groups[ID];
-        writeRead();
-        return true;
-    } else {
-        return false;
-    }
-}
-
-function getIeeeAddrByFriendlyName(friendlyName) {
-    const entry = Object.entries(getDevices()).find(([ieeeAddr, device]) =>
-        device.friendly_name === friendlyName
-    );
-    return entry && entry[0];
-}
-
-function getGroupIDByFriendlyName(friendlyName) {
-    const entry = Object.entries(getGroups()).find(([ID, group]) =>
-        group.friendly_name === friendlyName
-    );
-    return entry && entry[0];
-}
-
-function changeDeviceOptions(ieeeAddr, newOptions) {
-    const settings = getSettings();
-    const currentOptions = settings.devices[ieeeAddr];
-
-    if (!currentOptions) {
-        return;
-    }
-
-    Object.keys(currentOptions).forEach((key) => {
-        if (newOptions[key]) {
-            currentOptions[key] = newOptions[key];
-        }
-    });
-
-    writeRead();
-}
-
-function changeFriendlyName(old, new_) {
-    const settings = getSettings();
-    const ieeeAddr = getIeeeAddrByFriendlyName(old);
-
-    if (!ieeeAddr) {
-        return false;
-    }
-
-    settings.devices[ieeeAddr].friendly_name = new_;
-    writeRead();
-    return true;
-}
-
-// An entity can be either a group or a device.
-function resolveEntity(ID) {
-    let type = null;
-    let friendlyName = null;
-
-    if (module.exports.getIeeeAddrByFriendlyName(ID)) {
-        // Check if the ID is a friendly_name of a device.
-        friendlyName = ID;
-        ID = module.exports.getIeeeAddrByFriendlyName(ID);
-        type = 'device';
-    } else if (module.exports.getGroupIDByFriendlyName(ID)) {
-        // Check if the ID is a friendly_name of a group.
-        friendlyName = ID;
-        ID = Number(module.exports.getGroupIDByFriendlyName(ID));
-        type = 'group';
-    } else if (module.exports.getGroup(ID)) {
-        friendlyName = module.exports.getGroup(ID).friendly_name;
-        ID = Number(ID);
-        type = 'group';
-    } else {
-        // By default it is a device with ID as ID.
-        type = 'device';
-        const device = module.exports.getDevice(ID);
-        friendlyName = device ? device.friendly_name : ID;
-    }
-
-    return {ID, type, friendlyName};
-}
-
-module.exports = {
-    get: () => get(),
-    write: () => write(),
-    set: (path, value) => set(path, value),
-
-    getDevice,
-    getGroup,
-    getGroups,
-    getDevices,
-    whitelistDevice: (ieeeAddr) => whitelistDevice(ieeeAddr),
-    banDevice: (ieeeAddr) => banDevice(ieeeAddr),
-    addDevice: (ieeeAddr) => addDevice(ieeeAddr),
-    removeDevice: (ieeeAddr) => removeDevice(ieeeAddr),
-    addGroup: (name) => addGroup(name),
-    removeGroup: (name) => removeGroup(name),
-    addDeviceToGroup: (ID, ieeeAddr) => addDeviceToGroup(ID, ieeeAddr),
-    removeDeviceFromGroup: (ID, ieeeAddr) => removeDeviceFromGroup(ID, ieeeAddr),
-
-    getIeeeAddrByFriendlyName: (friendlyName) => getIeeeAddrByFriendlyName(friendlyName),
-    getGroupIDByFriendlyName: (friendlyName) => getGroupIDByFriendlyName(friendlyName),
-    changeFriendlyName: (old, new_) => changeFriendlyName(old, new_),
-    changeDeviceOptions: (ieeeAddr, options) => changeDeviceOptions(ieeeAddr, options),
-    resolveEntity,
-
-    addOnChangeHandler: (handler) => onChangeHandlers.push(handler),
-
-    // For test
-    _getDefaults: () => {
-        return objectAssignDeep.noMutate(defaults);
-    },
-    _clear: () => _settings = undefined,
-};

lib/util/settings.schema.json

@@ -0,0 +1,767 @@
+{
+  "type": "object",
+  "properties": {
+    "device_options": {
+      "type": "object"
+    },
+    "homeassistant": {
+      "title": "Home Assistant integration",
+      "type": "boolean",
+      "description": "Home Assistant integration (MQTT discovery)",
+      "default": false
+    },
+    "permit_join": {
+      "type": "boolean",
+      "default": false,
+      "title": "Permit join",
+      "description": "Allow new devices to join (re-applied at restart)"
+    },
+    "external_converters": {
+      "type": "array",
+      "title": "External converters",
+      "description": "You can define external converters to e.g. add support for a DiY device",
+      "requiresRestart": true,
+      "items": {
+        "type": "string"
+      },
+      "examples": ["DIYRuZ_FreePad.js"]
+    },
+    "mqtt": {
+      "type": "object",
+      "title": "MQTT",
+      "properties": {
+        "base_topic": {
+          "type": "string",
+          "title": "Base topic",
+          "requiresRestart": true,
+          "description": "MQTT base topic for Zigbee2MQTT MQTT messages",
+          "examples": ["zigbee2mqtt"]
+        },
+        "server": {
+          "type": "string",
+          "title": "MQTT server",
+          "requiresRestart": true,
+          "description": "MQTT server URL (use mqtts:// for SSL/TLS connection)",
+          "examples": ["mqtt://localhost:1883"]
+        },
+        "keepalive": {
+          "type": "number",
+          "title": "Keepalive",
+          "requiresRestart": true,
+          "description": "MQTT keepalive in second",
+          "default": 60
+        },
+        "ca": {
+          "type": "string",
+          "title": "Certificate authority",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS certificate of CA used to sign server and client certificates",
+          "examples": ["/etc/ssl/mqtt-ca.crt"]
+        },
+        "key": {
+          "type": "string",
+          "title": "SSL/TLS key",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS key for client-authentication",
+          "examples": ["/etc/ssl/mqtt-client.key"]
+        },
+        "cert": {
+          "type": "string",
+          "title": "SSL/TLS certificate",
+          "description": "Absolute path to SSL/TLS certificate for client-authentication",
+          "requiresRestart": true,
+          "examples": ["/etc/ssl/mqtt-client.crt"]
+        },
+        "user": {
+          "type": "string",
+          "title": "User",
+          "requiresRestart": true,
+          "description": "MQTT server authentication user",
+          "examples": ["johnnysilverhand"]
+        },
+        "password": {
+          "type": "string",
+          "title": "Password",
+          "requiresRestart": true,
+          "description": "MQTT server authentication password",
+          "examples": ["ILOVEPELMENI"]
+        },
+        "client_id": {
+          "type": "string",
+          "title": "Client ID",
+          "requiresRestart": true,
+          "description": "MQTT client ID",
+          "examples": ["MY_CLIENT_ID"]
+        },
+        "reject_unauthorized": {
+          "type": "boolean",
+          "title": "Reject unauthorized",
+          "requiresRestart": true,
+          "description": "Disable self-signed SSL certificate",
+          "default": true
+        },
+        "include_device_information": {
+          "type": "boolean",
+          "title": "Include device information",
+          "description": "Include device information to mqtt messages",
+          "default": false
+        },
+        "version": {
+          "type": ["number", "null"],
+          "title": "Version",
+          "requiresRestart": true,
+          "description": "MQTT protocol version",
+          "default": 4,
+          "examples": [5]
+        },
+        "force_disable_retain": {
+          "type": "boolean",
+          "title": "Force disable retain",
+          "requiresRestart": true,
+          "description": "Disable retain for all send messages. ONLY enable if you MQTT broker doesn't support retained message (e.g. AWS IoT core, Azure IoT Hub, Google Cloud IoT core, IBM Watson IoT Platform). Enabling will break the Home Assistant integration",
+          "default": false
+        }
+      },
+      "required": ["base_topic", "server"]
+    },
+    "serial": {
+      "type": "object",
+      "title": "Serial",
+      "properties": {
+        "port": {
+          "type": ["string", "null"],
+          "title": "Port",
+          "requiresRestart": true,
+          "description": "Location of the adapter. To autodetect the port, set null",
+          "examples": ["/dev/ttyACM0"]
+        },
+        "disable_led": {
+          "type": "boolean",
+          "title": "Disable led",
+          "requiresRestart": true,
+          "description": "Disable LED of the adapter if supported",
+          "default": false
+        },
+        "adapter": {
+          "type": ["string"],
+          "enum": ["deconz", "zstack", "zigate", "ezsp", "auto"],
+          "title": "Adapter",
+          "default": "auto",
+          "requiresRestart": true,
+          "description": "Adapter type, not needed unless you are experiencing problems"
+        }
+      }
+    },
+    "blocklist": {
+      "title": "Blocklist",
+      "requiresRestart": true,
+      "description": "Block devices from the network (by ieeeAddr)",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "passlist": {
+      "title": "Passlist",
+      "requiresRestart": true,
+      "description": "Allow only certain devices to join the network (by ieeeAddr). Note that all devices not on the passlist will be removed from the network!",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "whitelist": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Whitelist (deprecated, use passlist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "ban": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Ban (deprecated, use blocklist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "experimental": {
+      "type": "object",
+      "title": "Experimental",
+      "properties": {
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        }
+      }
+    },
+    "advanced": {
+      "type": "object",
+      "title": "Advanced",
+      "properties": {
+        "legacy_api": {
+          "type": "boolean",
+          "title": "Legacy API",
+          "requiresRestart": true,
+          "description": "Disables the legacy api (false = disable)",
+          "default": true
+        },
+        "pan_id": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Pan ID (string)"
+            },
+            {
+              "type": "number",
+              "title": "Pan ID (number)"
+            }
+          ],
+          "title": "Pan ID",
+          "requiresRestart": true,
+          "description": "ZigBee pan ID, changing requires repairing all devices!"
+        },
+        "ext_pan_id": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "title": "Ext Pan ID",
+          "requiresRestart": true,
+          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
+        },
+        "channel": {
+          "type": "number",
+          "minimum": 11,
+          "maximum": 26,
+          "default": 11,
+          "title": "ZigBee channel",
+          "requiresRestart": true,
+          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
+          "examples": [15, 20, 25]
+        },
+        "cache_state": {
+          "type": "boolean",
+          "title": "Cache state",
+          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
+          "default": true
+        },
+        "cache_state_persistent": {
+          "type": "boolean",
+          "title": "Persist cache state",
+          "description": "Persist cached state, only used when cache_state: true",
+          "default": true
+        },
+        "cache_state_send_on_startup": {
+          "type": "boolean",
+          "title": "Send cached state on startup",
+          "description": "Send cached state on startup, only used when cache_state: true",
+          "default": true
+        },
+        "log_rotation": {
+          "type": "boolean",
+          "title": "Log rotation",
+          "requiresRestart": true,
+          "description": "Log rotation",
+          "default": true
+        },
+        "log_symlink_current": {
+          "type": "boolean",
+          "title": "Log symlink current",
+          "requiresRestart": true,
+          "description": "Create symlink to current logs in the log directory",
+          "default": false
+        },
+        "log_level": {
+          "type": "string",
+          "enum": ["info", "warn", "error", "debug"],
+          "title": "Log level",
+          "description": "Logging level",
+          "default": "info"
+        },
+        "log_output": {
+          "type": "array",
+          "requiresRestart": true,
+          "items": {
+            "type": "string",
+            "enum": ["console", "file", "syslog"]
+          },
+          "title": "Log output",
+          "description": "Output location of the log, leave empty to supress logging"
+        },
+        "log_directory": {
+          "type": "string",
+          "title": "Log directory",
+          "requiresRestart": true,
+          "description": "Location of log directory",
+          "examples": ["data/log/%TIMESTAMP%"]
+        },
+        "log_file": {
+          "type": "string",
+          "title": "Log file",
+          "requiresRestart": true,
+          "description": "Log file name, can also contain timestamp",
+          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
+          "default": "log.txt"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
+        },
+        "soft_reset_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "requiresRestart": true,
+          "title": "Soft reset timeout (deprecated)",
+          "description": "Soft reset ZNP after timeout",
+          "readOnly": true
+        },
+        "network_key": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Network key(string)"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "number"
+              },
+              "title": "Network key(array)"
+            }
+          ],
+          "title": "Network key",
+          "requiresRestart": true,
+          "description": "Network encryption key, changing requires repairing all devices!"
+        },
+        "last_seen": {
+          "type": "string",
+          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
+          "title": "Last seen",
+          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
+          "default": "disable"
+        },
+        "elapsed": {
+          "type": "boolean",
+          "title": "Elapsed",
+          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
+          "default": false
+        },
+        "availability_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "default": 0,
+          "requiresRestart": true,
+          "title": "Availability Timeout",
+          "description": "Availability timeout in seconds when enabled, devices will be checked if they are still online. Only AC powered routers are checked for availability"
+        },
+        "availability_blocklist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability Blocklist",
+          "description": "Prevent devices from being checked for availability"
+        },
+        "availability_passlist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability passlist",
+          "description": "Only enable availability check for certain devices"
+        },
+        "availability_blacklist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "title": "Availability blacklist (deprecated, use availability_blocklist)",
+          "items": {
+            "type": "string"
+          }
+        },
+        "availability_whitelist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "items": {
+            "type": "string"
+          },
+          "title": "Availability whitelist (deprecated, use passlist)"
+        },
+        "report": {
+          "type": "boolean",
+          "title": "Reporting",
+          "requiresRestart": true,
+          "readOnly": true,
+          "description": "Enables report feature (deprecated)"
+        },
+        "homeassistant_discovery_topic": {
+          "type": "string",
+          "title": "Homeassistant discovery topic",
+          "description": "Home Assistant discovery topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant"]
+        },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
+        "homeassistant_status_topic": {
+          "type": "string",
+          "title": "Home Assistant status topic",
+          "description": "Home Assistant status topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant/status"]
+        },
+        "timestamp_format": {
+          "type": "string",
+          "title": "Timestamp format",
+          "requiresRestart": true,
+          "description": "Log timestamp format",
+          "examples": ["YYYY-MM-DD HH:mm:ss"]
+        },
+        "adapter_concurrent": {
+          "title": "Adapter concurrency",
+          "requiresRestart": true,
+          "type": ["number", "null"],
+          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
+        },
+        "adapter_delay": {
+          "type": ["number", "null"],
+          "requiresRestart": true,
+          "title": "Adapter delay",
+          "description": "Adapter delay"
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
+        },
+        "homeassistant_legacy_triggers": {
+          "type": "boolean",
+          "title": "Home Assistant legacy triggers",
+          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
+          "default": true
+        },
+        "log_syslog": {
+          "type": "object",
+          "title": "syslog",
+          "properties": {
+            "host": {
+              "type": "string",
+              "title": "Host",
+              "description": "The host running syslogd, defaults to localhost.",
+              "default": "localhost"
+            },
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
+              "default": 123
+            },
+            "protocol": {
+              "type": "string",
+              "title": "Protocol",
+              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
+              "default": "tcp4",
+              "examples": [
+                "udp4",
+                "tls4",
+                "unix",
+                "unix-connect"
+              ]
+            },
+            "path": {
+              "type": "string",
+              "title": "Path",
+              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
+              "default": "/dev/log",
+              "examples": [
+                "/var/run/syslog"
+              ]
+            },
+            "pid": {
+              "type": "string",
+              "title": "PID",
+              "description": "PID of the process that log messages are coming from (Default process.pid).",
+              "default": "process.pid"
+            },
+            "localhost": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "Host to indicate that log messages are coming from (Default: localhost).",
+              "default": "localhost"
+            },
+            "type": {
+              "type": "string",
+              "title": "Type",
+              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
+              "default": "5424"
+            },
+            "app_name": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "The name of the application (Default: Zigbee2MQTT).",
+              "default": "Zigbee2MQTT"
+            },
+            "eol": {
+              "type": "string",
+              "title": "eol",
+              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
+              "default": "/n"
+            }
+          }
+        }
+      }
+    },
+    "map_options": {
+      "type": "object",
+      "title": "Networkmap",
+      "properties": {
+        "graphviz": {
+          "type": "object",
+          "properties": {
+            "colors": {
+              "type": "object",
+              "properties": {
+                "fill": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "font": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "line": {
+                  "type": "object",
+                  "properties": {
+                    "active": {
+                      "type": "string"
+                    },
+                    "inactive": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "ota": {
+      "type": "object",
+      "title": "OTA updates",
+      "properties": {
+        "update_check_interval": {
+          "type": "number",
+          "title": "Update check interval",
+          "description": "Your device may request a check for a new firmware update. This value determines how frequently third party servers may actually be contacted to look for firmware updates. The value is set in minutes, and the default is 1 day.",
+          "default": 1440
+        },
+        "disable_automatic_update_check": {
+          "type": "boolean",
+          "title": "Disable automatic update check",
+          "description": "Zigbee devices may request a firmware update, and do so frequently, causing Zigbee2MQTT to reach out to third party servers. If you disable these device initiated checks, you can still initiate a firmware update check manually.",
+          "default": false
+        }
+      }
+    },
+    "devices": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^0x[\\d\\w]{16}$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/device"
+        }
+      }
+    },
+    "groups": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^[\\w].*$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/group"
+        }
+      }
+    },
+    "frontend": {
+      "type": "object",
+      "title": "Frontend",
+      "properties": {
+        "port": {
+          "type": "number",
+          "title": "Port",
+          "description": "Frontend binding port",
+          "default": 8080,
+          "requiresRestart": true
+        },
+        "host": {
+          "type": "string",
+          "title": "Bind host",
+          "description": "Frontend binding host",
+          "default":" 0.0.0.0",
+          "requiresRestart": true
+        },
+        "auth_token": {
+          "type": ["string", "null"],
+          "title": "Auth token",
+          "description": "Enables authentication, disabled by default",
+          "requiresRestart": true
+        }
+      }
+    }
+  },
+  "required": ["mqtt"],
+  "definitions": {
+    "device": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string",
+          "title": "Friendly name",
+          "description": "Used in the MQTT topic of a device. By default this is the device ID",
+          "readOnly": true
+        },
+        "retain": {
+          "type": "boolean",
+          "title": "Retain",
+          "description": "Retain MQTT messages of this device"
+        },
+        "retention": {
+          "type": "number",
+          "title": "Retention",
+          "description": "Sets the MQTT Message Expiry in seconds, Make sure to set mqtt.version to 5"
+        },
+        "qos": {
+          "type": "number",
+          "title": "QoS",
+          "description": "QoS level for MQTT messages of this device"
+        },
+        "debounce": {
+          "type": "number",
+          "title": "Debounce",
+          "description": "Debounces messages of this device"
+        },
+        "debounce_ignore": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["action"],
+          "title": "Ignore debounce",
+          "description": "Protects unique payload values of specified payload properties from overriding within debounce time"
+        },
+        "optimistic": {
+          "type": "boolean",
+          "title": "Optimistic",
+          "description": "Publish optimistic state after set",
+          "default": true
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["color_mode", "color_temp", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["temperature", "battery", "action"],
+          "title": "Filtered publish attributes",
+          "description": "Filter attributes from publish payload."
+        },
+        "icon": {
+          "type": "string",
+          "title": "Icon",
+          "description": "The user-defined device icon for the frontend. It can be a link to an image (not a path to a file) or base64 encoded data URL like: image/svg+xml;base64,PHN2ZyB3aW....R0aD"
+        }
+      },
+      "required": ["friendly_name"]
+    },
+    "group": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string"
+        },
+        "retain": {
+          "type": "boolean"
+        },
+        "devices": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "optimistic": {
+          "type": "boolean"
+        },
+        "qos": {
+          "type": "number"
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": ["friendly_name"]
+    }
+  }
+}

lib/util/settings.ts

@@ -0,0 +1,710 @@
+import * as data from './data';
+import * as utils from './utils';
+import objectAssignDeep from 'object-assign-deep';
+import path from 'path';
+import * as yaml from './yaml';
+import Ajv from 'ajv';
+import schemaJson from './settings.schema.json';
+export const schema = schemaJson;
+
+// TODO: check all
+
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
+const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+
+const defaults: RecursivePartial<Settings> = {
+    passlist: [],
+    blocklist: [],
+    // Deprecated: use block/passlist
+    whitelist: [],
+    ban: [],
+    permit_join: false,
+    mqtt: {
+        include_device_information: false,
+        /**
+         * Configurable force disable retain flag on mqtt publish.
+         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
+         */
+        force_disable_retain: false,
+    },
+    serial: {
+        disable_led: false,
+    },
+    device_options: {},
+    map_options: {
+        graphviz: {
+            colors: {
+                fill: {
+                    enddevice: '#fff8ce',
+                    coordinator: '#e04e5d',
+                    router: '#4ea3e0',
+                },
+                font: {
+                    coordinator: '#ffffff',
+                    router: '#ffffff',
+                    enddevice: '#000000',
+                },
+                line: {
+                    active: '#009900',
+                    inactive: '#994444',
+                },
+            },
+        },
+    },
+    experimental: {
+        // json or attribute or attribute_and_json
+        output: 'json',
+    },
+    advanced: {
+        legacy_api: true,
+        log_rotation: true,
+        log_symlink_current: false,
+        log_output: ['console', 'file'],
+        log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
+        log_file: 'log.txt',
+        log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
+        soft_reset_timeout: 0,
+        pan_id: 0x1a62,
+        ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
+        channel: 11,
+        adapter_concurrent: null,
+        adapter_delay: null,
+
+        // Availability timeout in seconds, disabled by default.
+        availability_timeout: 0,
+        availability_blocklist: [],
+        availability_passlist: [],
+        // Deprecated, use block/passlist
+        availability_blacklist: [],
+        availability_whitelist: [],
+
+        /**
+         * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
+         * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
+         *
+         * Therefore Zigbee2MQTT BY DEFAULT caches all values and resend it with every message.
+         * advanced.cache_state in configuration.yaml allows to configure this.
+         * https://www.zigbee2mqtt.io/configuration/configuration.html
+         */
+        cache_state: true,
+        cache_state_persistent: true,
+        cache_state_send_on_startup: true,
+
+        /**
+         * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
+         * "ISO_8601": ISO 8601 format
+         * "ISO_8601_local": Local ISO 8601 format (instead of UTC-based)
+         * "epoch": milliseconds elapsed since the UNIX epoch
+         * "disable": no last_seen attribute (default)
+         */
+        last_seen: 'disable',
+
+        // Optional: Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg
+        elapsed: false,
+
+        /**
+         * https://github.com/Koenkk/zigbee2mqtt/issues/685#issuecomment-449112250
+         *
+         * Network key will serve as the encryption key of your network.
+         * Changing this will require you to repair your devices.
+         */
+        network_key: [1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13],
+
+        /**
+         * Enables reporting feature
+         */
+        report: false,
+
+        /**
+         * Home Assistant discovery topic
+         */
+        homeassistant_discovery_topic: 'homeassistant',
+
+        /**
+         * Home Assistant status topic
+         */
+        homeassistant_status_topic: 'hass/status',
+
+        /**
+         * Home Assistant legacy entity attributes, when enabled:
+         * Zigbee2MQTT will send additional states as attributes with each entity.
+         * For example, A temperature & humidity sensor will have 2 entities for
+         * the temperature and humidity, with this setting enabled both entities
+         * will also have an temperature and humidity attribute.
+         */
+        homeassistant_legacy_entity_attributes: true,
+
+        /**
+         * Home Assistant legacy triggers, when enabled:
+         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
+         * - A 'sensor_action' and 'sensor_click' will be discoverd
+         */
+        homeassistant_legacy_triggers: true,
+
+        /**
+         * Configurable timestampFormat
+         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
+         */
+        timestamp_format: 'YYYY-MM-DD HH:mm:ss',
+    },
+    ota: {
+        /**
+         * Minimal time delta in minutes between polling third party server for potential firmware updates
+         */
+        update_check_interval: 24 * 60,
+        /**
+         * Completely disallow Zigbee devices to initiate a search for a potential firmware update.
+         * If set to true, only a user-initiated update search will be possible.
+         */
+        disable_automatic_update_check: false,
+    },
+    external_converters: [],
+};
+
+let _settings: Partial<Settings>;
+let _settingsWithDefaults: Settings;
+
+function write(): void {
+    const settings = getInternalSettings();
+    /* eslint-disable-line */ // @ts-ignore
+    const toWrite = objectAssignDeep.noMutate(settings);
+
+    // Read settings to check if we have to split devices/groups into separate file.
+    const actual = yaml.read(file);
+
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [
+        ['mqtt', 'user'],
+        ['mqtt', 'password'],
+        ['advanced', 'network_key'],
+        ['frontend', 'auth_token'],
+    ]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
+    }
+
+    // Write devices/groups to separate file if required.
+    const writeDevicesOrGroups = (type: 'devices' | 'groups'): void => {
+        if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
+            const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            /* eslint-disable-line */ // @ts-ignore
+            const content = objectAssignDeep.noMutate(settings[type]);
+
+            // If an array, only write to first file and only devices which are not in the other files.
+            if (Array.isArray(actual[type])) {
+                actual[type].filter((f: string, i: number) => i !== 0)
+                    .map((f: string) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c: KeyValue) => Object.keys(c))
+                    .forEach((k: string) => delete content[k]);
+            }
+
+            yaml.writeIfChanged(data.joinPath(fileToWrite), content);
+            toWrite[type] = actual[type];
+        }
+    };
+
+    writeDevicesOrGroups('devices');
+    writeDevicesOrGroups('groups');
+
+    yaml.writeIfChanged(file, toWrite);
+
+    _settings = read();
+    /* eslint-disable-line */ // @ts-ignore
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
+}
+
+export function validate(): string[] {
+    try {
+        getInternalSettings();
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            return [
+                `Your YAML file: '${error.file}' is invalid ` +
+                `(use https://jsonformatter.org/yaml-validator to find and fix the issue)`,
+            ];
+        }
+
+        return [error.message];
+    }
+
+    if (!ajvSetting(_settings)) {
+        return ajvSetting.errors.map((v) => `${v.instancePath.substring(1)} ${v.message}`);
+    }
+
+    const errors = [];
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        errors.push(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
+    if (_settings.advanced && _settings.advanced.pan_id && typeof _settings.advanced.pan_id === 'string' &&
+        _settings.advanced.pan_id !== 'GENERATE') {
+        errors.push(`advanced.pan_id: should be number or 'GENERATE' (is '${_settings.advanced.pan_id}')`);
+    }
+
+    // Verify that all friendly names are unique
+    const names: string[] = [];
+    const check = (name: string): void => {
+        if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
+        errors.push(...utils.validateFriendlyName(name));
+        names.push(name);
+    };
+
+    const settingsWithDefaults = get();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
+            if (device.retention) {
+                errors.push('MQTT retention requires protocol version 5');
+            }
+        }
+    }
+
+    const checkAvailabilityList = (list: string[], type: string): void => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                errors.push(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blocklist, 'availability_blocklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_passlist, 'availability_passlist');
+
+    return errors;
+}
+
+function read(): Settings {
+    const s = yaml.read(file) as Settings;
+
+    // Read !secret MQTT username and password if set
+    // eslint-disable-next-line
+    const interpetValue = (value: any): any => {
+        const re = /!(.*) (.*)/g;
+        const match = re.exec(value);
+        if (match) {
+            const file = data.joinPath(`${match[1]}.yaml`);
+            const key = match[2];
+            return yaml.read(file)[key];
+        } else {
+            return value;
+        }
+    };
+
+    if (s.mqtt?.user && s.mqtt?.password) {
+        s.mqtt.user = interpetValue(s.mqtt.user);
+        s.mqtt.password = interpetValue(s.mqtt.password);
+    }
+
+    if (s.advanced?.network_key) {
+        s.advanced.network_key = interpetValue(s.advanced.network_key);
+    }
+
+    if (s.frontend?.auth_token) {
+        s.frontend.auth_token = interpetValue(s.frontend.auth_token);
+    }
+
+    // Read devices/groups configuration from separate file if specified.
+    const readDevicesOrGroups = (type: 'devices' | 'groups'): void => {
+        if (typeof s[type] === 'string' || Array.isArray(s[type])) {
+            /* eslint-disable-line */ // @ts-ignore
+            const files: string[] = Array.isArray(s[type]) ? s[type] : [s[type]];
+            s[type] = {};
+            for (const file of files) {
+                const content = yaml.readIfExists(data.joinPath(file), {});
+                /* eslint-disable-line */ // @ts-ignore
+                s[type] = objectAssignDeep.noMutate(s[type], content);
+            }
+        }
+    };
+
+    readDevicesOrGroups('devices');
+    readDevicesOrGroups('groups');
+
+    return s;
+}
+
+function applyEnvironmentVariables(settings: Partial<Settings>): void {
+    const iterate = (obj: KeyValue, path: string[]): void => {
+        Object.keys(obj).forEach((key) => {
+            if (key !== 'type') {
+                if (key !== 'properties' && obj[key]) {
+                    const type = (obj[key].type || 'object').toString();
+                    const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
+                    const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
+                    if (process.env[envVariableName]) {
+                        const setting = path.reduce((acc, val) => {
+                            /* eslint-disable-line */ // @ts-ignore
+                            acc[val] = acc[val] || {};
+                            /* eslint-disable-line */ // @ts-ignore
+                            return acc[val];
+                        }, settings);
+
+                        if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
+                            setting[key] = JSON.parse(process.env[envVariableName]);
+                        } else if (type.indexOf('number') >= 0) {
+                            /* eslint-disable-line */ // @ts-ignore
+                            setting[key] = process.env[envVariableName] * 1;
+                        } else if (type.indexOf('boolean') >= 0) {
+                            setting[key] = process.env[envVariableName].toLowerCase() === 'true';
+                        } else {
+                            /* istanbul ignore else */
+                            if (type.indexOf('string') >= 0) {
+                                setting[key] = process.env[envVariableName];
+                            }
+                        }
+                    }
+                }
+
+                if (typeof obj[key] === 'object' && obj[key]) {
+                    const newPath = [...path];
+                    if (key !== 'properties') {
+                        newPath.push(key);
+                    }
+                    iterate(obj[key], newPath);
+                }
+            }
+        });
+    };
+    iterate(schema.properties, []);
+}
+
+function getInternalSettings(): Partial<Settings> {
+    if (!_settings) {
+        _settings = read();
+        applyEnvironmentVariables(_settings);
+    }
+
+    return _settings;
+}
+
+export function get(): Settings {
+    if (!_settingsWithDefaults) {
+        /* eslint-disable-line */ // @ts-ignore
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
+    }
+
+    if (!_settingsWithDefaults.devices) {
+        _settingsWithDefaults.devices = {};
+    }
+
+    if (!_settingsWithDefaults.groups) {
+        _settingsWithDefaults.groups = {};
+    }
+
+    return _settingsWithDefaults;
+}
+
+export function set(path: string[], value: string | number | boolean | KeyValue): void {
+    /* eslint-disable-next-line */
+    let settings: any = getInternalSettings();
+
+    for (let i = 0; i < path.length; i++) {
+        const key = path[i];
+        if (i === path.length - 1) {
+            settings[key] = value;
+        } else {
+            if (!settings[key]) {
+                settings[key] = {};
+            }
+
+            settings = settings[key];
+        }
+    }
+
+    write();
+}
+
+export function apply(newSettings: Record<string, unknown>): boolean {
+    ajvSetting(newSettings);
+    const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
+    if (errors.length) {
+        const error = errors[0];
+        throw new Error(`${error.instancePath.substring(1)} ${error.message}`);
+    }
+
+    getInternalSettings(); // Ensure _settings is intialized.
+    /* eslint-disable-line */ // @ts-ignore
+    _settings = objectAssignDeep.noMutate(_settings, newSettings);
+    write();
+
+    ajvRestartRequired(newSettings);
+    const restartRequired = ajvRestartRequired.errors &&
+        !!ajvRestartRequired.errors.find((e) => e.keyword === 'requiresRestart');
+    return restartRequired;
+}
+
+export function getGroup(IDorName: string | number): GroupSettings {
+    const settings = get();
+    const byID = settings.groups[IDorName];
+    if (byID) {
+        return {devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, group] of Object.entries(settings.groups)) {
+        if (group.friendly_name === IDorName) {
+            return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+export function getGroups(): GroupSettings[] {
+    const settings = get();
+    return Object.entries(settings.groups).map(([ID, group]) => {
+        return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+    });
+}
+
+function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
+    const group = getGroup(IDorName);
+    if (!group) {
+        throw new Error(`Group '${IDorName}' does not exist`);
+    }
+
+    return group;
+}
+
+export function getDevice(IDorName: string): DeviceSettings {
+    const settings = get();
+    const byID = settings.devices[IDorName];
+    if (byID) {
+        return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, device] of Object.entries(settings.devices)) {
+        if (device.friendly_name === IDorName) {
+            return {...device, ID, friendlyName: device.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
+    const device = getDevice(IDorName);
+    if (!device) {
+        throw new Error(`Device '${IDorName}' does not exist`);
+    }
+
+    return device;
+}
+
+export function getEntity(IDorName: string): EntitySettings {
+    const device = getDevice(IDorName);
+    if (device) {
+        return {...device, type: 'device'};
+    }
+
+    const group = getGroup(IDorName);
+    if (group) {
+        return {...group, type: 'group'};
+    }
+
+    return null;
+}
+
+export function addDevice(ID: string): DeviceSettings {
+    if (getDevice(ID)) {
+        throw new Error(`Device '${ID}' already exists`);
+    }
+
+    const settings = getInternalSettings();
+
+    if (!settings.devices) {
+        settings.devices = {};
+    }
+
+    settings.devices[ID] = {friendly_name: ID};
+    write();
+    return getDevice(ID);
+}
+
+export function whitelistDevice(ID: string): void {
+    const settings = getInternalSettings();
+    if (!settings.whitelist) {
+        settings.whitelist = [];
+    }
+
+    if (settings.whitelist.includes(ID)) {
+        throw new Error(`Device '${ID}' already whitelisted`);
+    }
+
+    settings.whitelist.push(ID);
+    write();
+}
+
+export function blockDevice(ID: string): void {
+    const settings = getInternalSettings();
+    if (!settings.blocklist) {
+        settings.blocklist = [];
+    }
+
+    settings.blocklist.push(ID);
+    write();
+}
+
+export function banDevice(ID: string): void {
+    const settings = getInternalSettings();
+    if (!settings.ban) {
+        settings.ban = [];
+    }
+
+    settings.ban.push(ID);
+    write();
+}
+
+export function removeDevice(IDorName: string): void {
+    const device = getDeviceThrowIfNotExists(IDorName);
+    const settings = getInternalSettings();
+    delete settings.devices[device.ID];
+
+    // Remove device from groups
+    if (settings.groups) {
+        const regex =
+            new RegExp(`^(${device.friendlyName}|${device.ID})(/(\\d|${utils.endpointNames.join('|')}))?$`);
+        for (const group of Object.values(settings.groups).filter((g) => g.devices)) {
+            group.devices = group.devices.filter((device) => !device.match(regex));
+        }
+    }
+
+    write();
+}
+
+export function addGroup(name: string, ID?: string): GroupSettings {
+    utils.validateFriendlyName(name, true);
+    if (getGroup(name) || getDevice(name)) {
+        throw new Error(`friendly_name '${name}' is already in use`);
+    }
+
+    const settings = getInternalSettings();
+    if (!settings.groups) {
+        settings.groups = {};
+    }
+
+    if (ID == null) {
+        // look for free ID
+        ID = '1';
+        while (settings.groups.hasOwnProperty(ID)) {
+            ID = (Number.parseInt(ID) + 1).toString();
+        }
+    } else {
+        // ensure provided ID is not in use
+        ID = ID.toString();
+        if (settings.groups.hasOwnProperty(ID)) {
+            throw new Error(`Group ID '${ID}' is already in use`);
+        }
+    }
+
+    settings.groups[ID] = {friendly_name: name};
+    write();
+
+    return getGroup(ID);
+}
+
+function groupGetDevice(group: {devices?: string[]}, keys: string[]): string {
+    for (const device of group.devices ?? []) {
+        if (keys.includes(device)) return device;
+    }
+
+    return null;
+}
+
+export function addDeviceToGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
+
+    const group = settings.groups[groupID];
+    if (!groupGetDevice(group, keys)) {
+        if (!group.devices) group.devices = [];
+        group.devices.push(keys[0]);
+        write();
+    }
+}
+
+export function removeDeviceFromGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
+    const group = settings.groups[groupID];
+    if (!group.devices) {
+        return;
+    }
+
+    const key = groupGetDevice(group, keys);
+    if (key) {
+        group.devices = group.devices.filter((d) => d != key);
+        write();
+    }
+}
+
+export function removeGroup(IDorName: string | number): void {
+    const groupID = getGroupThrowIfNotExists(IDorName.toString()).ID;
+    const settings = getInternalSettings();
+    delete settings.groups[groupID];
+    write();
+}
+
+export function changeEntityOptions(IDorName: string, newOptions: KeyValue): void {
+    const settings = getInternalSettings();
+    delete newOptions.friendly_name;
+    delete newOptions.devices;
+    if (getDevice(IDorName)) {
+        objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.devices[getDevice(IDorName).ID]);
+    } else if (getGroup(IDorName)) {
+        objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.groups[getGroup(IDorName).ID]);
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
+    write();
+}
+
+export function changeFriendlyName(IDorName: string, newName: string): void {
+    utils.validateFriendlyName(newName, true);
+    if (getGroup(newName) || getDevice(newName)) {
+        throw new Error(`friendly_name '${newName}' is already in use`);
+    }
+
+    const settings = getInternalSettings();
+    if (getDevice(IDorName)) {
+        settings.devices[getDevice(IDorName).ID].friendly_name = newName;
+    } else if (getGroup(IDorName)) {
+        settings.groups[getGroup(IDorName).ID].friendly_name = newName;
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
+    write();
+}
+
+export function reRead(): void {
+    _settings = null;
+    getInternalSettings();
+    _settingsWithDefaults = null;
+    get();
+}
+
+export const testing = {
+    write,
+    clear: (): void => {
+        _settings = null;
+        _settingsWithDefaults = null;
+    },
+    defaults,
+};

lib/util/utils.js

@@ -1,152 +0,0 @@
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
-
-// Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
-const xiaomiManufacturerID = [4151, 4447];
-const ikeaTradfriManufacturerID = [4476];
-
-// construct a local ISO8601 string (instead of UTC-based)
-// Example:
-//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
-//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
-function toLocalISOString(dDate) {
-    const tzOffset = -dDate.getTimezoneOffset();
-    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
-    const pad = function(num) {
-        const norm = Math.floor(Math.abs(num));
-        return (norm < 10 ? '0' : '') + norm;
-    };
-
-    return dDate.getFullYear() +
-        '-' + pad(dDate.getMonth() + 1) +
-        '-' + pad(dDate.getDate()) +
-        'T' + pad(dDate.getHours()) +
-        ':' + pad(dDate.getMinutes()) +
-        ':' + pad(dDate.getSeconds()) +
-        plusOrMinus + pad(tzOffset / 60) +
-        ':' + pad(tzOffset % 60);
-}
-
-const postfixes = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'l1', 'l2',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom', 'center_left', 'center_right',
-];
-
-function flatten(arr) {
-    return arr.reduce((flat, toFlatten) => {
-        return flat.concat(Array.isArray(toFlatten) ? flatten(toFlatten) : toFlatten);
-    }, []);
-}
-
-const forceEndDevice = flatten(
-    ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM']
-        .map((model) => zigbeeShepherdConverters.devices.find((d) => d.model === model))
-        .map((mappedModel) => mappedModel.zigbeeModel));
-
-function getEndpointByEntityID(zigbee, entityID, epName) {
-    const device = zigbee.getDevice(entityID);
-    if (!device) {
-        logger.error(`Failed to find device with entity ID '${entityID}'`);
-        return;
-    }
-
-    const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-    if (!mappedDevice) {
-        logger.error(`Device with model ID ${device.modelId} is not supported`);
-        return;
-    }
-
-    let epID = null;
-    if (epName) {
-        if (!mappedDevice.ep) {
-            logger.error(`Device ${mappedDevice.model} doesn't define eps`);
-            return;
-        }
-
-        epID = mappedDevice.ep(device)[epName];
-        if (!epID) {
-            logger.error(`Device ${mappedDevice.model} doesn't have ep named '${epName}'`);
-            return;
-        }
-    } else if (mappedDevice.hasOwnProperty('ep')) {
-        const eps = mappedDevice.ep(device);
-        epID = eps[''] || null;
-    }
-
-    const endpoint = zigbee.getEndpoint(entityID, epID);
-    if (!endpoint) {
-        logger.error(`Failed to retrieve for entity ID ${entityID} and endpoint ID ${epID}`);
-    }
-
-    return endpoint;
-}
-
-function getZigbee2mqttVersion(callback) {
-    const git = require('git-last-commit');
-    const packageJSON = require('../../package.json');
-    const version = packageJSON.version;
-
-    git.getLastCommit((err, commit) => {
-        let commitHash = null;
-
-        if (err) {
-            try {
-                commitHash = require('../../.hash.json').hash;
-            } catch (error) {
-                commitHash = 'unknown';
-            }
-        } else {
-            commitHash = commit.shortHash;
-        }
-
-        callback({commitHash, version});
-    });
-}
-
-function formatDate(date, type, _default) {
-    let result;
-
-    switch (type) {
-    case 'ISO_8601':
-        result = new Date(date).toISOString();
-        break;
-    case 'ISO_8601_local':
-        result = toLocalISOString(new Date(date));
-        break;
-    case 'epoch':
-        result = date;
-        break;
-    default:
-        result = _default;
-        break;
-    }
-
-    return result;
-}
-
-module.exports = {
-    millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
-    secondsToMilliseconds: (seconds) => seconds * 1000,
-    isXiaomiDevice: (device) => {
-        return device.modelId !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufId) &&
-            (!device.manufName || !device.manufName.startsWith('Trust'));
-    },
-    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufId),
-    isRouter: (device) => device.type === 'Router' && !forceEndDevice.includes(device.modelId),
-    isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
-    isNumeric: (string) => /^\d+$/.test(string),
-    toLocalISOString: (dDate) => toLocalISOString(dDate),
-    getPostfixes: () => postfixes,
-    formatDate: (date, type, _default=null) => formatDate(date, type, _default),
-    correctDeviceType: (device) => {
-        if (device) {
-            if (forceEndDevice.includes(device.modelId)) {
-                return 'EndDevice';
-            }
-        }
-
-        return device.type;
-    },
-    getEndpointByEntityID,
-    getZigbee2mqttVersion,
-};

lib/util/utils.ts

@@ -0,0 +1,328 @@
+import equals from 'fast-deep-equal/es6';
+import humanizeDuration from 'humanize-duration';
+import * as data from './data';
+import vm from 'vm';
+import fs from 'fs';
+import path from 'path';
+import {Endpoint} from 'zigbee-herdsman/dist/controller/model';
+
+// TODO: check all
+
+// construct a local ISO8601 string (instead of UTC-based)
+// Example:
+//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
+//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
+function toLocalISOString(date: Date): string {
+    const tzOffset = -date.getTimezoneOffset();
+    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
+    const pad = (num: number): string => {
+        const norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+    };
+
+    return date.getFullYear() +
+        '-' + pad(date.getMonth() + 1) +
+        '-' + pad(date.getDate()) +
+        'T' + pad(date.getHours()) +
+        ':' + pad(date.getMinutes()) +
+        ':' + pad(date.getSeconds()) +
+        plusOrMinus + pad(tzOffset / 60) +
+        ':' + pad(tzOffset % 60);
+}
+
+export const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay', 'usb',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
+    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
+    'heat', 'cool', 'water', 'meter', 'wifi',
+];
+
+export function capitalize(s: string): string {
+    return s[0].toUpperCase() + s.slice(1);
+}
+
+export async function getZigbee2MQTTVersionSimple(): Promise<string> {
+    const packageJSON = await import('../..' + '/package.json');
+    return packageJSON.version;
+}
+
+export async function getZigbee2MQTTVersion(): Promise<{commitHash: string, version: string}> {
+    const git = await import('git-last-commit');
+    const packageJSON = await import('../..' + '/package.json');
+
+    return new Promise((resolve) => {
+        const version = packageJSON.version;
+
+        git.getLastCommit((err: Error, commit: {shortHash: string}) => {
+            let commitHash = null;
+
+            if (err) {
+                try {
+                    commitHash = fs.readFileSync(path.join(__dirname, '..', '..', 'dist', '.hash'), 'utf-8');
+                } catch (error) {
+                    /* istanbul ignore next */
+                    commitHash = 'unknown';
+                }
+            } else {
+                commitHash = commit.shortHash;
+            }
+
+            resolve({commitHash, version});
+        });
+    });
+}
+
+export async function getDependencyVersion(depend: string): Promise<{version: string}> {
+    const packageJSON = await import(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
+    const version = packageJSON.version;
+    return {version};
+}
+
+export function formatDate(time: number, type: 'ISO_8601' | 'ISO_8601_local' | 'epoch' | 'relative'): string | number {
+    if (type === 'ISO_8601') return new Date(time).toISOString();
+    else if (type === 'ISO_8601_local') return toLocalISOString(new Date(time));
+    else if (type === 'epoch') return time;
+    else { // relative
+        return humanizeDuration(Date.now() - time, {language: 'en', largest: 2, round: true}) + ' ago';
+    }
+}
+
+export function objectHasProperties(object: {[s: string]: unknown}, properties: string[]): boolean {
+    for (const property of properties) {
+        if (!object.hasOwnProperty(property)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+export function equalsPartial(object: {[s: string]: unknown}, expected: {[s: string]: unknown}): boolean {
+    for (const [key, value] of Object.entries(expected)) {
+        if (!equals(object[key], value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+export function getObjectProperty(object: {[s: string]: unknown}, key: string, defaultValue: unknown): unknown {
+    return object && object.hasOwnProperty(key) ? object[key] : defaultValue;
+}
+
+export function getResponse(request: KeyValue | string, data: KeyValue, error: string): MQTTResponse {
+    const response: {data: unknown, status: string, error?: string, transaction?: string} =
+        {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
+export function parseJSON(value: string, failedReturnValue: string): KeyValue | string {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return failedReturnValue;
+    }
+}
+
+export function loadModuleFromText(moduleCode: string): unknown {
+    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+    const sandbox = {
+        require: require,
+        module: {},
+        console,
+        setTimeout,
+        clearTimeout,
+        setInterval,
+        clearInterval,
+        setImmediate,
+        clearImmediate,
+    };
+    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    /* eslint-disable-line */ // @ts-ignore
+    return sandbox.module.exports;
+}
+
+export function loadModuleFromFile(modulePath: string): unknown {
+    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
+    return loadModuleFromText(moduleCode);
+}
+
+/* eslint-disable-next-line */
+export function* getExternalConvertersDefinitions(settings: any): any {
+    const externalConverters = settings.get().external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converter;
+
+        if (moduleName.endsWith('.js')) {
+            converter = loadModuleFromFile(data.joinPath(moduleName));
+        } else {
+            converter = require(moduleName);
+        }
+
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+export function removeNullPropertiesFromObject(obj: KeyValue): void {
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (value == null) {
+            delete obj[key];
+        } else if (typeof value === 'object') {
+            removeNullPropertiesFromObject(value);
+        }
+    }
+}
+
+export function getKey(
+    object: KeyValue, value: unknown, fallback: unknown, convertTo: (v: unknown) => unknown): unknown {
+    for (const key in object) {
+        if (object[key]===value) {
+            return convertTo ? convertTo(key) : key;
+        }
+    }
+
+    return fallback;
+}
+
+export function toNetworkAddressHex(value: number): string {
+    const hex = value.toString(16);
+    return `0x${'0'.repeat(4 - hex.length)}${hex}`;
+}
+
+// eslint-disable-next-line
+export function toSnakeCase(value: string | KeyValue): any {
+    if (typeof value === 'object') {
+        value = {...value};
+        for (const key of Object.keys(value)) {
+            const keySnakeCase = toSnakeCase(key);
+            if (key !== keySnakeCase) {
+                value[keySnakeCase] = value[key];
+                delete value[key];
+            }
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
+export function validateFriendlyName(name: string, throwFirstError=false): string[] {
+    const errors = [];
+    for (const endpointName of endpointNames) {
+        if (name.toLowerCase().endsWith('/' + endpointName)) {
+            errors.push(`friendly_name is not allowed to end with: '/${endpointName}'`);
+        }
+    }
+
+    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
+    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
+    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
+    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
+    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+    if (name.includes('#') || name.includes('+')) {
+        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+    }
+
+    if (throwFirstError && errors.length) {
+        throw new Error(errors[0]);
+    }
+
+    return errors;
+}
+
+export function sleep(seconds: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+export function sanitizeImageParameter(parameter: string): string {
+    const replaceByDash = [/\?/g, /&/g, /[^a-z\d\- _./:]/gi];
+    let sanitized = parameter;
+    replaceByDash.forEach((r) => sanitized = sanitized.replace(r, '-'));
+    return sanitized;
+}
+
+export function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boolean {
+    /* istanbul ignore next */
+    if (!settings.experimental.availability_new) return false;
+
+    if (device.settings.hasOwnProperty('availability')) {
+        return !!device.settings.availability;
+    }
+
+    // availability_timeout = deprecated
+    const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
+    if (!enabledGlobal) return false;
+
+    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
+    if (passlist.length > 0) {
+        return passlist.includes(device.name) || passlist.includes(device.ieeeAddr);
+    }
+
+    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
+    return !blocklist.includes(device.name) && !blocklist.includes(device.ieeeAddr);
+}
+
+/* istanbul ignore next */
+export function isAvailabilityEnabledForDeviceLegacy(rd: ResolvedDevice, settings: Settings): boolean {
+    if (!settings.experimental.availability_new) return false;
+
+    if (rd.settings.hasOwnProperty('availability')) {
+        return !!rd.settings.availability;
+    }
+
+    // availability_timeout = deprecated
+    const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
+    if (!enabledGlobal) return false;
+
+    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
+    if (passlist.length > 0) {
+        return passlist.includes(rd.name) || passlist.includes(rd.device.ieeeAddr);
+    }
+
+    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
+    return !blocklist.includes(rd.name) && !blocklist.includes(rd.device.ieeeAddr);
+}
+
+export function isXiaomiDevice(device: ZHDevice): boolean {
+    const xiaomiManufacturerID = [4151, 4447];
+    return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
+        (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
+}
+
+export function isIkeaTradfriDevice(device: ZHDevice): boolean {
+    return [4476].includes(device.manufacturerID);
+}
+
+const entityIDRegex = new RegExp(`^(.+?)(?:/(${endpointNames.join('|')}|\\d+))?$`);
+export function parseEntityID(ID: string): {ID: string, endpoint: string} {
+    const match = ID.match(entityIDRegex);
+    return match && {ID: match[1], endpoint: match[2]};
+}
+
+export function isEndpoint(obj: unknown): obj is Endpoint {
+    return obj.constructor.name.toLowerCase() === 'endpoint';
+}
+
+export const hours = (hours: number): number => 1000 * 60 * 60 * hours;
+export const minutes = (minutes: number): number => 1000 * 60 * minutes;
+export const seconds = (seconds: number): number => 1000 * seconds;

lib/util/yaml.ts

@@ -0,0 +1,36 @@
+import yaml from 'js-yaml';
+import fs from 'fs';
+import equals from 'fast-deep-equal/es6';
+
+// TODO: check all
+export function read(file: string): KeyValue {
+    try {
+        /* eslint-disable-line */ // @ts-ignore
+        return yaml.load(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.file = file;
+        }
+
+        throw error;
+    }
+}
+
+export function readIfExists(file: string, default_?: KeyValue): KeyValue {
+    return fs.existsSync(file) ? read(file) : default_;
+}
+
+export function writeIfChanged(file: string, content: KeyValue): void {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.dump(content));
+    }
+}
+
+export function updateIfChanged(file: string, key: string, value: KeyValue): void {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}

lib/util/zigbeeQueue.js

@@ -1,97 +0,0 @@
-const settings = require('./settings');
-
-const maxSimultaneouslyRunning = settings.get().queue.simultaneously;
-const delay = settings.get().queue.delay;
-const error17Retries = 3;
-const logger = require('./logger');
-
-class ZigbeeQueue {
-    constructor() {
-        this.queue = [];
-        this.active = [];
-        this.timer = null;
-        this.running = false;
-        this.ID = 0;
-    }
-
-    start() {
-        this.running = true;
-        this.resetTimer();
-    }
-
-    push(entityID, func) {
-        this.ID++;
-        this.queue.push({entityID, func, attempts: 0, ID: this.ID});
-        this.log(`Added new job with ID ${this.ID} for ${entityID}`);
-    }
-
-    log(message) {
-        logger.trace(`zigbeeQueue: ${message}`);
-    }
-
-    stopTimer() {
-        if (this.timer) {
-            clearInterval(this.timer);
-            this.timer = null;
-        }
-    }
-
-    resetTimer() {
-        this.stopTimer();
-
-        this.timer = setInterval(() => {
-            this.executeNext();
-        }, delay);
-    }
-
-    stop() {
-        this.stopTimer();
-        this.running = false;
-    }
-
-    handleJobComplete(job, error) {
-        this.log(`Completed job with ID ${job.ID} for ${job.entityID}${error ? ` with error ${error}` : ''}`);
-
-        if (error && error.message === 'rsp error: 17' && job.attempts < error17Retries) {
-            // Error 17 means that the buffer of the ZNP was full,
-            // retry this for a maximum of 3 times.
-            job.attempts++;
-            this.active.splice(this.active.indexOf(job), 1);
-        } else {
-            this.active.splice(this.active.indexOf(job), 1);
-            this.queue.splice(this.queue.indexOf(job), 1);
-        }
-    }
-
-    executeNext() {
-        if (!this.running) {
-            return;
-        }
-
-        const next = this.getNext();
-
-        if (next) {
-            this.active.push(next);
-            this.log(`Executing job with ID ${next.ID} for ${next.entityID}`);
-            next.func((error) => this.handleJobComplete(next, error));
-        }
-    }
-
-    getNext() {
-        if (this.active.length > (maxSimultaneouslyRunning - 1)) {
-            return null;
-        }
-
-        for (let i = 0; i < this.queue.length; i++) {
-            const job = this.queue[i];
-            const activeDeviceJob = this.active.find((j) => j.entityID === job.entityID);
-            if (!activeDeviceJob) {
-                return job;
-            }
-        }
-
-        return null;
-    }
-}
-
-module.exports = ZigbeeQueue;

lib/zapp/cie.js

@@ -1,23 +0,0 @@
-/**
- * This is a fake cie app which makes it possible for some iAS devices to join the network.
- * Based on: https://github.com/zigbeer/zapp-cie
- */
-
-const Ziee = require('zigbee-herdsman/dist/deprecated/ziee');
-const Zive = require('zigbee-herdsman/dist/deprecated/zive');
-const cieClusters = new Ziee();
-// Direction-Value 1: input, 2: output, 3: input/output
-
-// IasZone Cluster-Init Direction
-cieClusters.init('ssIasZone', 'dir', {value: 2}); // Client Side(Output)
-
-// Init Command Response Handler
-cieClusters.init('ssIasZone', 'cmdRsps', {
-    enrollReq: (zapp, argObj, cb) => {},
-    statusChangeNotification: (zapp, argObj, cb) => {},
-});
-
-// Create Zive with profId: 0x0104 (Decimal: 260 / HA), devId: 0x0400 (Decimal: 1024 / iasControlIndicatingEquipment)
-// Check the Values by https://github.com/zigbeer/zcl-id/wiki#5-table-of-identifiers
-const cieApp = new Zive({profId: 0x0104, devId: 0x0400, discCmds: []}, cieClusters);
-module.exports = cieApp;

lib/zigbee.js

@@ -1,663 +0,0 @@
-const ZShepherd = require('zigbee-herdsman');
-const logger = require('./util/logger');
-const settings = require('./util/settings');
-const data = require('./util/data');
-const utils = require('./util/utils');
-const ZigbeeQueue = require('./util/zigbeeQueue');
-const cieApp = require('./zapp/cie');
-const objectAssignDeep = require('object-assign-deep');
-const Zcl = require('zigbee-herdsman/dist/zcl');
-
-const advancedSettings = settings.get().advanced;
-
-if (advancedSettings.channel < 11 || advancedSettings.channel > 26) {
-    throw new Error(`'${advancedSettings.channel}' is an invalid channel, use a channel between 11 - 26.`);
-}
-
-const shepherdSettings = {
-    net: {
-        panId: advancedSettings.pan_id,
-        extPanId: advancedSettings.ext_pan_id,
-        channelList: [advancedSettings.channel],
-        precfgkey: settings.get().advanced.network_key,
-    },
-    dbPath: data.joinPath('database.db'),
-    coordBackupPath: data.joinPath('coordinator_backup.json'),
-    sp: {
-        baudRate: advancedSettings.baudrate,
-        rtscts: advancedSettings.rtscts,
-    },
-};
-
-const defaultCfg = {
-    manufSpec: 0,
-    disDefaultRsp: 0,
-};
-
-// Don't print network key.
-const shepherdSettingsLog = objectAssignDeep.noMutate(shepherdSettings);
-shepherdSettingsLog.net.precfgkey = 'HIDDEN';
-
-class Zigbee {
-    constructor() {
-        this.onReady = this.onReady.bind(this);
-        this.onMessage = this.onMessage.bind(this);
-        this.onError = this.onError.bind(this);
-        this.messageHandler = null;
-        this.permitJoinTimer = null;
-
-        this.queue = new ZigbeeQueue();
-    }
-
-    start(messageHandler, callback) {
-        logger.info(`Starting zigbee-shepherd`);
-        logger.debug(`Using zigbee-shepherd with settings: '${JSON.stringify(shepherdSettingsLog)}'`);
-
-        this.messageHandler = messageHandler;
-        this.shepherd = new ZShepherd(settings.get().serial.port, shepherdSettings);
-
-        this.shepherd.start((error) => {
-            if (error) {
-                logger.info(`Error while starting zigbee-shepherd, attempting to fix... (takes 60 seconds) (${error})`);
-                this.shepherd.controller._znp.close((() => null));
-
-                setTimeout(() => {
-                    logger.info(`Starting zigbee-shepherd`);
-                    this.shepherd.start((error) => {
-                        if (error) {
-                            logger.error(`Error while starting zigbee-shepherd! (${error})`);
-                            logger.error(
-                                'Press the reset button on the stick (the one closest to the USB) and start again'
-                            );
-                            callback(error);
-                        } else {
-                            this._handleStarted();
-                            callback(null);
-                        }
-                    });
-                }, utils.secondsToMilliseconds(60));
-            } else {
-                this._handleStarted();
-                callback(null);
-            }
-        });
-
-        // Register callbacks.
-        this.shepherd.on('ready', this.onReady);
-        this.shepherd.on('ind', this.onMessage);
-        this.shepherd.on('error', this.onError);
-        this._acceptDevIncoming = this._acceptDevIncoming.bind(this);
-        this.shepherd.acceptDevIncoming = this._acceptDevIncoming;
-    }
-
-    _handleStarted() {
-        this.logStartupInfo();
-
-        this.getAllClients().forEach((device) => {
-            // If set whitelist devices, all other device will be ban or reject to join the network
-            if (settings.get().whitelist.size>0) {
-                if (!settings.get().whitelist.includes(device.ieeeAddr)) {
-                    logger.warn(`Blacklist device is connected (${device.ieeeAddr}), removing...`);
-                    this.removeDevice(device.ieeeAddr, true, () => {});
-                }
-            } else {
-                if (settings.get().ban.includes(device.ieeeAddr)) {
-                    logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
-                    this.removeDevice(device.ieeeAddr, false, () => {});
-                }
-            }
-        });
-
-        this.shepherd.backupCoordinator(() => {});
-    }
-
-    _acceptDevIncoming(devInfo, callback) {
-        logger.debug(
-            `Accept device incoming with ieeeAddr '${devInfo.ieeeAddr}' permit join is '${this.getPermitJoin()}'`
-        );
-        // If set whitelist devices, all other device will be ban or reject to join the network
-        if (settings.get().whitelist.size>0) {
-            if (settings.get().whitelist.includes(devInfo.ieeeAddr)) {
-                logger.info(`whitelist device tried to connect (${devInfo.ieeeAddr})`);
-                callback(null, true);
-            } else {
-                logger.debug(`Not allowing device '${devInfo.ieeeAddr}' to join`);
-                callback(null, false);
-            }
-        } else {
-            if (settings.get().ban.includes(devInfo.ieeeAddr)) {
-                logger.info(`Banned device tried to connect (${devInfo.ieeeAddr})`);
-                callback(null, false);
-            } else {
-                logger.debug(`Allowing device '${devInfo.ieeeAddr}' to join`);
-                callback(null, true);
-            }
-        }
-    }
-
-    logStartupInfo() {
-        logger.info('zigbee-shepherd started');
-        logger.info(`Coordinator firmware version: '${this.getFirmwareVersion()}'`);
-        logger.debug(`zigbee-shepherd info: ${JSON.stringify(this.shepherd.info())}`);
-    }
-
-    getFirmwareVersion() {
-        return this.shepherd.info().firmware.revision;
-    }
-
-    softReset(callback) {
-        this.shepherd.reset('soft', callback);
-    }
-
-    stop(callback) {
-        if (this.permitJoinTimer) {
-            clearInterval(this.permitJoinTimer);
-        }
-
-        this.queue.stop();
-
-        // Backup coordinator
-        this.shepherd.backupCoordinator(() => {
-            this.shepherd.stop((error) => {
-                logger.info('zigbee-shepherd stopped');
-                callback(error);
-            });
-        });
-    }
-
-    onReady() {
-        // Mount cieApp
-        this.shepherd.mount(cieApp, (err, epId) => {
-            if (!err) {
-                logger.debug(`Mounted the cieApp (epId ${epId})`);
-            } else {
-                logger.error(`Failed to mount the cieApp`);
-            }
-        });
-
-        // Check if we have to turn off the led
-        if (settings.get().serial.disable_led) {
-            this.shepherd.controller.request('UTIL', 'ledControl', {ledid: 3, mode: 0});
-        }
-
-        // Wait some time before we start the queue, many calls skip this queue which hangs the stick
-        setTimeout(() => {
-            this.queue.start();
-        }, 2000);
-
-        logger.info('zigbee-shepherd ready');
-    }
-
-    onError(message) {
-        // This event may appear if zigbee-shepherd cannot decode bad packets (invalid checksum).
-        logger.error(message);
-    }
-
-    permitJoin(permit, callback) {
-        if (permit) {
-            logger.info('Zigbee: allowing new devices to join.');
-        } else {
-            logger.info('Zigbee: disabling joining new devices.');
-        }
-
-        // In zigbee 3.0 a network automatically closes after 254 seconds.
-        // As a workaround, we enable joining again.
-        if (this.permitJoinTimer) {
-            clearInterval(this.permitJoinTimer);
-        }
-
-        if (permit) {
-            this.permitJoinTimer = setInterval(() => {
-                this.shepherd.permitJoin(255, (error) => {
-                    if (error) {
-                        logger.error('Failed to reenable joining');
-                    } else {
-                        logger.info('Successfully reenabled joining');
-                    }
-                });
-            }, utils.secondsToMilliseconds(160));
-        }
-
-        this.shepherd.permitJoin(permit ? 255 : 0, (error) => {
-            if (error) {
-                logger.info(error);
-            }
-
-            if (callback) {
-                callback();
-            }
-        });
-    }
-
-    getPermitJoin() {
-        return this.shepherd.controller._permitJoinTime === 255;
-    }
-
-    getAllClients() {
-        return this.getDevices().filter((device) => device.type !== 'Coordinator');
-    }
-
-    removeDevice(deviceID, ban, callback) {
-        if (ban) {
-            settings.banDevice(deviceID);
-        }
-
-        const friendlyName = this.getDeviceFriendlyName(deviceID);
-
-        this.shepherd.remove(deviceID, {reJoin: true}, (error) => {
-            if (error) {
-                logger.warn(`Failed to remove '${friendlyName}', trying force remove...`);
-                this.forceRemove(deviceID, callback);
-            } else {
-                logger.info(`Removed ${friendlyName}`);
-                callback(null);
-            }
-        });
-    }
-
-    forceRemove(deviceID, callback) {
-        const device = this.shepherd._findDevByAddr(deviceID);
-
-        if (device) {
-            const friendlyName = this.getDeviceFriendlyName(deviceID);
-            return this.shepherd._unregisterDev(device, (error) => {
-                logger.info(`Force removed ${friendlyName}`);
-                callback(error);
-            });
-        } else {
-            logger.warn(`Could not find ${deviceID} for force removal`);
-            callback(true);
-        }
-    }
-
-    onMessage(message) {
-        if (this.messageHandler) {
-            this.messageHandler(message);
-        }
-    }
-
-    getDevices() {
-        return this.shepherd.list();
-    }
-
-    getDevice(ieeeAddr) {
-        return this.getDevices().find((d) => d.ieeeAddr === ieeeAddr);
-    }
-
-    getDeviceFriendlyName(ieeeAddr) {
-        const device = settings.getDevice(ieeeAddr);
-        return device ? device.friendly_name || ieeeAddr : ieeeAddr;
-    }
-
-    getCoordinator() {
-        const device = this.getDevices().find((d) => d.type === 'Coordinator');
-        return this.shepherd.find(device.ieeeAddr, 1);
-    }
-
-    getScanable() {
-        return this.getDevices().filter((d) => d.type != 'EndDevice');
-    }
-
-    getGroup(ID) {
-        return this.shepherd.getGroup(ID);
-    }
-
-    getGroupFriendlyName(ID) {
-        let friendlyName = null;
-        friendlyName = settings.getGroup(ID).friendly_name;
-        return (friendlyName ? friendlyName : ID);
-    }
-
-    networkScan(includeRoutes, callback) {
-        logger.info(`Starting network scan includeRoutes='${includeRoutes}'...`);
-        const lqiScanList = new Set();
-        const rtgScanList = new Set();
-        const linkMap = [];
-        const routeMap = [];
-
-        // Gather the lqi and the route info into separate lists and only collate them when done.
-        const collateMap = () => {
-            linkMap.sort();
-            logger.debug(`Link map: %j`, linkMap);
-            logger.debug(`Route map: %j`, routeMap);
-            // Merge the routes into the linkMap by matching on 'source|target' link short addresses
-            linkMap.forEach((link) => {
-                routeMap.filter((e) => e.key === link.key).forEach((e) => {
-                    link.routes.push(e.destAddr);
-                });
-                link.routes.sort(function(a, b) {
-                    return a-b;
-                });
-                delete link.key;
-            });
-            const networkMap = {nodes: [], links: linkMap};
-            this.getDevices().forEach((device) => {
-                const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                const friendlyName = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
-                const deviceType = utils.correctDeviceType(device);
-                const scanfailed = [];
-                if (lqiScanList.has(device.ieeeAddr)) {
-                    scanfailed.push('lqi');
-                }
-                if (rtgScanList.has(device.ieeeAddr)) {
-                    scanfailed.push('rtg');
-                }
-                networkMap.nodes.push({ieeeAddr: device.ieeeAddr, friendlyName: friendlyName, type: deviceType,
-                    nwkAddr: device.nwkAddr, manufName: device.manufName, modelId: device.modelId,
-                    status: device.status, scanfailed: scanfailed});
-            });
-            // Clear remaining devices so they don't process when/if they eventually complete
-            lqiScanList.clear();
-            rtgScanList.clear();
-            logger.debug(`Merged map: %j`, networkMap);
-            callback(networkMap);
-        };
-
-        const processLqiResponse = (error, rsp, targetIeeeAddr, targetNwkAddr) => {
-            if (error) {
-                logger.warn(`Failed network lqi scan for device: '${targetIeeeAddr}' with error: '${error}'`);
-            } else {
-                if (lqiScanList.has(targetIeeeAddr)) {
-                    // Haven't processed this one yet
-                    if (rsp && rsp.status === 0 && rsp.neighborlqilist) {
-                        logger.debug(`lqi scan: '${targetIeeeAddr}' with '${rsp.neighborlqilistcount}' neighbors`);
-                        rsp.neighborlqilist.forEach(function(neighbor) {
-                            // only include active relationships
-                            if (neighbor.relationship <= 3) {
-                                // lqi is measured at receiver so link is from neighbor (source) to scanned router
-                                const key = neighbor.nwkAddr + '|' + targetNwkAddr;
-                                linkMap.push({
-                                    key: key, sourceIeeeAddr: neighbor.extAddr, targetIeeeAddr: targetIeeeAddr,
-                                    sourceNwkAddr: neighbor.nwkAddr, lqi: neighbor.lqi, depth: neighbor.depth,
-                                    relationship: neighbor.relationship, routes: []});
-                            }
-                        });
-                        // Remove from scan list and if both lists are done return the completed network map
-                        lqiScanList.delete(targetIeeeAddr);
-                        if (lqiScanList.size === 0 && rtgScanList.size === 0) {
-                            logger.info('Network scan completed');
-                            collateMap();
-                        } else {
-                            logger.debug(`Outstanding network lqi scans for devices: '${[...lqiScanList].join(' ')}'`);
-                            logger.debug(`Outstanding network rtg scans for devices: '${[...rtgScanList].join(' ')}'`);
-                        }
-                    } else {
-                        logger.warn(`Empty network lqi scan result for: '${targetIeeeAddr}'`);
-                    }
-                } else {
-                    // This target has already had timeout so don't add to result network map
-                    logger.warn(`Ignoring late network lqi scan result for: '${targetIeeeAddr}'`);
-                }
-            }
-        };
-
-        const processRtgResponse = (error, rsp, sourceIeeeAddr, sourceNwkAddr) => {
-            if (error) {
-                logger.warn(`Failed network rtg scan for device: '${sourceIeeeAddr}' with error: '${error}'`);
-            } else {
-                if (rtgScanList.has(sourceIeeeAddr)) {
-                    // Haven't processed this one yet
-                    if (rsp && rsp.status === 0 && rsp.routingtablelist) {
-                        logger.debug(`rtg scan: '${sourceIeeeAddr}' with '${rsp.routingtablelistcount}' entries`);
-                        rsp.routingtablelist.forEach(function(route) {
-                            if (route.routeStatus === 0) {
-                                const key = sourceNwkAddr + '|' + route.nextHopNwkAddr;
-                                routeMap.push({
-                                    key: key, destAddr: route.destNwkAddr});
-                            }
-                        });
-                        // Remove from scan list and if both lists are done return the completed network map
-                        rtgScanList.delete(sourceIeeeAddr);
-                        if (lqiScanList.size === 0 && rtgScanList.size === 0) {
-                            logger.info('Network scan completed');
-                            collateMap();
-                        } else {
-                            logger.debug(`Outstanding network lqi scans for devices: '${[...lqiScanList].join(' ')}'`);
-                            logger.debug(`Outstanding network rtg scans for devices: '${[...rtgScanList].join(' ')}'`);
-                        }
-                    } else {
-                        logger.warn(`Empty network rtg scan result for: '${sourceIeeeAddr}'`);
-                    }
-                } else {
-                    // This source has already had timeout so don't add to result network map
-                    logger.warn(`Ignoring late network rtg scan result for: '${sourceIeeeAddr}'`);
-                }
-            }
-        };
-
-        // Queue up an lqi scan and an rtg scan for coordinator and each router
-        this.getScanable().forEach((dev) => {
-            logger.debug(`Queing network scans for device: '${dev.ieeeAddr}'`);
-            lqiScanList.add(dev.ieeeAddr);
-            this.queue.push(dev.ieeeAddr, (queueCallback) => {
-                this.shepherd.controller.request('ZDO', 'mgmtLqiReq', {dstaddr: dev.nwkAddr, startindex: 0},
-                    (error, rsp, ieeeAddr, nwkAddr) => {
-                        processLqiResponse(error, rsp, dev.ieeeAddr, dev.nwkAddr);
-                        queueCallback(error);
-                    });
-            });
-            if (includeRoutes) {
-                rtgScanList.add(dev.ieeeAddr);
-                this.queue.push(dev.ieeeAddr, (queueCallback) => {
-                    this.shepherd.controller.request('ZDO', 'mgmtRtgReq', {dstaddr: dev.nwkAddr, startindex: 0},
-                        (error, rsp, ieeeAddr, nwkAddr) => {
-                            processRtgResponse(error, rsp, dev.ieeeAddr, dev.nwkAddr);
-                            queueCallback(error);
-                        });
-                });
-            }
-        });
-
-        // Wait for all device scans before forcing map with whatever results are already in
-        setTimeout(() => {
-            if (lqiScanList.size === 0 && rtgScanList.size === 0) {
-                logger.info('Network scan timeout no outstanding requests');
-            } else {
-                logger.warn(`Network scan timeout, skipping outstanding lqi scans for '${[...lqiScanList].join(' ')}'`);
-                logger.warn(`Network scan timeout, skipping outstanding rtg scans for '${[...rtgScanList].join(' ')}'`);
-                collateMap();
-            }
-        }, lqiScanList.size * 1000);
-    }
-
-    getEndpoint(ieeeAddr, ep) {
-        // If no ep is given, the first endpoint will be returned
-        // Find device in zigbee-shepherd
-        const device = this.getDevice(ieeeAddr);
-        if (!device || !device.epList || !device.epList.length) {
-            logger.error(`Zigbee cannot determine endpoint for '${ieeeAddr}'`);
-            return null;
-        }
-
-        ep = ep ? ep : device.epList[0];
-        const endpoint = this.shepherd.find(ieeeAddr, ep);
-        return endpoint;
-    }
-
-    publish(entityID, entityType, cid, cmd, cmdType, zclData, cfg=defaultCfg, ep, callback) {
-        let entity = null;
-        let friendlyName = null;
-        if (entityType === 'device') {
-            entity = this.getEndpoint(entityID, ep);
-            friendlyName = this.getDeviceFriendlyName(entityID);
-        } else if (entityType === 'group') {
-            entity = this.getGroup(entityID);
-            friendlyName = this.getGroupFriendlyName(entityID);
-        }
-
-        if (!entity) {
-            logger.error(
-                `Cannot publish message to ${entityType} because '${entityID}' is not known by zigbee-shepherd`
-            );
-            return;
-        }
-
-        this.queue.push(entityID, (queueCallback) => {
-            logger.info(
-                `Zigbee publish to ${entityType} '${friendlyName}', ${cid} - ${cmd} - ` +
-                `${JSON.stringify(zclData)} - ${JSON.stringify(cfg)} - ${entity.epId}`
-            );
-
-            const callback_ = (error, rsp) => {
-                if (error) {
-                    logger.error(
-                        `Zigbee publish to ${entityType} '${friendlyName}', ${cid} ` +
-                        `- ${cmd} - ${JSON.stringify(zclData)} ` +
-                        `- ${JSON.stringify(cfg)} - ${ep} ` +
-                        `failed with error ${error}`);
-                }
-
-                if (callback) {
-                    callback(error, rsp);
-                }
-
-                queueCallback(error);
-            };
-
-            if (cmdType === 'functional' && entity.functional) {
-                entity.functional(cid, cmd, zclData, cfg, callback_);
-            } else if (cmdType === 'foundation' && entity.foundation) {
-                entity.foundation(cid, cmd, zclData, cfg, callback_);
-            } else {
-                logger.error(`Unknown zigbee publish cmdType ${cmdType}`);
-            }
-        });
-    }
-
-    ping(ieeeAddr, errorLogLevel='error', cb, mechanism='default') {
-        const friendlyName = this.getDeviceFriendlyName(ieeeAddr);
-        const callback = (error) => {
-            if (error) {
-                logger[errorLogLevel](`Failed to ping '${friendlyName}'`);
-            } else {
-                logger.debug(`Successfully pinged '${friendlyName}'`);
-            }
-
-            if (cb) {
-                cb(error);
-            }
-        };
-
-        if (mechanism === 'default') {
-            const device = this.shepherd._findDevByAddr(ieeeAddr);
-            if (device) {
-                logger.debug(`Ping ${ieeeAddr} (default)`);
-                this.queue.push(ieeeAddr, (queueCallback) => {
-                    this.shepherd.controller.checkOnline(device, (error) => {
-                        callback(error);
-                        queueCallback(error);
-                    });
-                });
-            }
-        } else if (mechanism === 'basic') {
-            const endpoint = this.getEndpoint(ieeeAddr, null);
-            if (endpoint) {
-                logger.debug(`Ping ${ieeeAddr} (basic)`);
-                this.queue.push(ieeeAddr, (queueCallback) => {
-                    endpoint.foundation('genBasic', 'read', [{attrId: 0}], (error) => {
-                        callback(error);
-                        queueCallback(error);
-                    });
-                });
-            }
-        }
-    }
-
-    bind(ep, cluster, target, callback) {
-        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
-        const log = ` '${friendlyName}' - ${cluster}`;
-        target = !target ? this.getCoordinator() : target;
-
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Binding ${log}`);
-            ep.bind(cluster, target, (error) => {
-                if (error) {
-                    logger.error(`Failed to bind ${log} - (${error})`);
-                } else {
-                    logger.debug(`Successfully bound ${log}`);
-                }
-
-                callback(error);
-                queueCallback(error);
-            });
-        });
-    }
-
-    unbind(ep, cluster, target, callback) {
-        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
-        const log = ` '${friendlyName}' - ${cluster}`;
-        target = !target ? this.getCoordinator() : target;
-
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Unbinding ${log}`);
-            ep.unbind(cluster, target, (error) => {
-                if (error) {
-                    logger.error(`Failed to unbind ${log} - (${error})`);
-                } else {
-                    logger.debug(`Successfully unbound ${log}`);
-                }
-
-                callback(error);
-                queueCallback(error);
-            });
-        });
-    }
-
-    /*
-     * Setup reporting.
-     * Attributes is an array of attribute objects.
-     * each attribute object should contain the following properties:
-     *     attr    the attribute name,
-     *     min     the minimal time between reports in seconds,
-     *     max     the maximum time between reports in seconds,
-     *     change  the minimum amount of change before sending a report
-     */
-    report(ep, cluster, attributes) {
-        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
-        const cfgArr = attributes.map((attribute) => {
-            const attrId = Zcl.getAttributeLegacy(cluster, attribute.attr).value;
-            const dataType = Zcl.getAttributeTypeLegacy(cluster, attribute.attr).value;
-            return {
-                direction: 0,
-                attrId,
-                dataType,
-                minRepIntval: attribute.min,
-                maxRepIntval: attribute.max,
-                repChange: attribute.change,
-            };
-        });
-
-        const log=`for '${friendlyName}' - ${cluster} - ${attributes.length}`;
-
-        const configReport = () => {
-            this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-                ep.foundation(cluster, 'configReport', cfgArr, defaultCfg, (error) => {
-                    if (error) {
-                        logger.error(`Failed to setup reporting ${log} - (${error})`);
-                    } else {
-                        logger.debug(`Successfully setup reporting ${log}`);
-                    }
-
-                    queueCallback(error);
-                });
-            });
-        };
-
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Setup reporting ${log}`);
-
-            ep.bind(cluster, this.getCoordinator(), (error) => {
-                if (error) {
-                    logger.error(`Failed to bind for reporting ${log} - (${error})`);
-                } else {
-                    // Only if binding succeeds, setting-up reporting makes sense.
-                    configReport();
-                }
-
-                queueCallback(error);
-            });
-        });
-    }
-}
-
-module.exports = Zigbee;